mirror of
https://github.com/ilyakooo0/haskell.nix.git
synced 2024-09-11 06:55:40 +03:00
Supporting ca-derivations
experimental feature (#1494)
* Supporting `ca-derivations` experimental feature * Improved `contentAddressed.include` option description * Ensure deterministic output for content addressed components * Comment transformed into a Nix comment To avoid possibly useless rebuilds * Use `modules` arg `contentAddressed` flag * Tutorial about CA derivations Co-authored-by: Hamish Mackenzie <Hamish.K.Mackenzie@gmail.com>
This commit is contained in:
parent
2a89c68994
commit
2c1701b681
@ -50,6 +50,7 @@ let self =
|
|||||||
, writeHieFiles ? component.writeHieFiles
|
, writeHieFiles ? component.writeHieFiles
|
||||||
|
|
||||||
, ghcOptions ? component.ghcOptions
|
, ghcOptions ? component.ghcOptions
|
||||||
|
, contentAddressed ? component.contentAddressed
|
||||||
|
|
||||||
# Options for Haddock generation
|
# Options for Haddock generation
|
||||||
, doHaddock ? component.doHaddock # Enable haddock and hoogle generation
|
, doHaddock ? component.doHaddock # Enable haddock and hoogle generation
|
||||||
@ -316,7 +317,13 @@ let
|
|||||||
componentDrv = drv;
|
componentDrv = drv;
|
||||||
};
|
};
|
||||||
|
|
||||||
drv = stdenv.mkDerivation (commonAttrs // {
|
contentAddressedAttrs = lib.optionalAttrs contentAddressed {
|
||||||
|
__contentAddressed = true;
|
||||||
|
outputHashMode = "recursive";
|
||||||
|
outputHashAlgo = "sha256";
|
||||||
|
};
|
||||||
|
|
||||||
|
drv = stdenv.mkDerivation (commonAttrs // contentAddressedAttrs // {
|
||||||
pname = nameOnly;
|
pname = nameOnly;
|
||||||
inherit (package.identifier) version;
|
inherit (package.identifier) version;
|
||||||
|
|
||||||
@ -405,7 +412,17 @@ let
|
|||||||
(lib.optionalString stdenv.hostPlatform.isWindows ''
|
(lib.optionalString stdenv.hostPlatform.isWindows ''
|
||||||
export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
|
export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
|
||||||
'') +
|
'') +
|
||||||
(if stdenv.hostPlatform.isGhcjs then ''
|
# The following could be refactored but would lead to many rebuilds
|
||||||
|
|
||||||
|
# In case of content addressed components we need avoid parallel building (passing -j1)
|
||||||
|
# in order to have a deterministic output and therefore avoid potential situations
|
||||||
|
# where the binary cache becomes useless
|
||||||
|
# See also https://gitlab.haskell.org/ghc/ghc/-/issues/12935
|
||||||
|
(if contentAddressed then ''
|
||||||
|
runHook preBuild
|
||||||
|
$SETUP_HS build ${haskellLib.componentTarget componentId} -j1 ${lib.concatStringsSep " " setupBuildFlags}
|
||||||
|
runHook postBuild
|
||||||
|
'' else if stdenv.hostPlatform.isGhcjs then ''
|
||||||
runHook preBuild
|
runHook preBuild
|
||||||
# https://gitlab.haskell.org/ghc/ghc/issues/9221
|
# https://gitlab.haskell.org/ghc/ghc/issues/9221
|
||||||
$SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
|
$SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
|
||||||
|
65
docs/tutorials/ca-derivations.md
Normal file
65
docs/tutorials/ca-derivations.md
Normal file
@ -0,0 +1,65 @@
|
|||||||
|
# Content addressed derivations
|
||||||
|
## Introduction
|
||||||
|
|
||||||
|
Floating content addressed derivations (from now *CA derivations*) is an experimental feature which substantially change how the hashes in the store paths are calculated.
|
||||||
|
Indeed, normally derivations are input addressed i.e. the outputs store paths depends only on the derivation inputs, instead with CA derivations they depend on the content of the outputs.
|
||||||
|
|
||||||
|
This has two main advantages:
|
||||||
|
|
||||||
|
- The so-called "early cutoff", namely the ability of Nix to stop a build if the build outputs would be something already built.
|
||||||
|
For example suppose you add a comment in an Haskell source, at this point Nix will rebuild the component depending on this source but since the output will be the same (adding a comment is an "output-invariant" change for `ghc`) every other component that depends on that will not be rebuilt.
|
||||||
|
- Users of the same Nix store does not need to trust each other when using substituters.
|
||||||
|
|
||||||
|
You can find more information in the [ca-derivations page on the wiki](https://nixos.wiki/wiki/Ca-derivations) (and in the other resources linked there).
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
### Enable CA derivations in your system
|
||||||
|
First of all your Nix installation must support the `ca-derivations` experimental feature, this can done by adding the following in your `nix.conf`:
|
||||||
|
|
||||||
|
```
|
||||||
|
experimental-features = ca-derivations
|
||||||
|
```
|
||||||
|
|
||||||
|
Or if you use NixOS:
|
||||||
|
```
|
||||||
|
nix.extraOptions = ''
|
||||||
|
experimental-features = ca-derivations
|
||||||
|
'';
|
||||||
|
```
|
||||||
|
|
||||||
|
## Enable CA derivations in your project
|
||||||
|
At this point you can pass a new module to `project'` that tells `haskell.nix` to build every component in the project as CA derivation.
|
||||||
|
|
||||||
|
```
|
||||||
|
haskell-nix.project' {
|
||||||
|
# ...
|
||||||
|
|
||||||
|
modules = [{
|
||||||
|
contentAddressed = true;
|
||||||
|
# packages.project-name.components.exes.executable.contentAddressed = true;
|
||||||
|
}];
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
Optionally you can also specify which components you don't want to be content addressed.
|
||||||
|
|
||||||
|
## Known problems
|
||||||
|
### Limitation of the current CA derivations implementation
|
||||||
|
|
||||||
|
As explained in the [RFC 62](https://github.com/tweag/rfcs/blob/cas-rfc/rfcs/0062-content-addressed-paths.md)
|
||||||
|
|
||||||
|
> The current implementation has a naive approach that just forbids fetching a path if the local system has a different realisation for the same drv output. This approach is simple and correct, but it's possible that it might not be good-enough in practice as it can result in a totally useless binary cache in some pathological cases.
|
||||||
|
|
||||||
|
For example, suppose that your machine builds a derivation `A` producing an output `A.out` in your store and that after that a CI machine builds the same derivation `A` but producing a different output `A.out'` and populating a cache with this output.
|
||||||
|
At this point, if you need to build a derivation `B` that depends on `A`, since you already have the realisation `A.out` in your local store and you can't get `B.out` from the cache and you will end up building `B` even if one of its realisation is in the cache.
|
||||||
|
|
||||||
|
This means that, in some cases, enabling CA derivations would lead to more rebuilds than not having it.
|
||||||
|
|
||||||
|
### Hydra
|
||||||
|
Hydra currently doesn't support CA derivations, efforts are being made in this direction.
|
||||||
|
|
||||||
|
|
||||||
|
### GHC is not deterministic
|
||||||
|
Currently `ghc` is determinstic only disabling the parallel building i.e. passing `-j1`. [Here](https://gitlab.haskell.org/ghc/ghc/-/issues/12935) the upstream issue.
|
||||||
|
|
||||||
|
Having a deterministic `ghc` would be a dream since it will automatically fix all the pathological cases about substituters discussed above and would allow `haskell.nix` to parallel build even when using CA derivations.
|
@ -244,6 +244,14 @@ let
|
|||||||
type = listOfFilteringNulls str;
|
type = listOfFilteringNulls str;
|
||||||
default = def.ghcOptions or [];
|
default = def.ghcOptions or [];
|
||||||
};
|
};
|
||||||
|
contentAddressed = mkOption {
|
||||||
|
type = bool;
|
||||||
|
default = (def.contentAddressed or false);
|
||||||
|
description = ''
|
||||||
|
Build content addressed derivation, requires Nix to have experimental feature
|
||||||
|
`ca-derivations` enabled.
|
||||||
|
'';
|
||||||
|
};
|
||||||
planned = mkOption {
|
planned = mkOption {
|
||||||
description = "Set to true by `plan-to-nix` for any component that was included in the `plan.json` file.";
|
description = "Set to true by `plan-to-nix` for any component that was included in the `plan.json` file.";
|
||||||
# This is here so that (rather than in componentOptions) so it can be set project wide for stack projects
|
# This is here so that (rather than in componentOptions) so it can be set project wide for stack projects
|
||||||
|
57
test/ca-derivations-include/default.nix
Normal file
57
test/ca-derivations-include/default.nix
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
# Build a project enabling content addressed derivations for
|
||||||
|
# only a subset of the components
|
||||||
|
{ stdenv, pkgs, lib, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name, CADerivationsEnabled }:
|
||||||
|
|
||||||
|
with lib;
|
||||||
|
|
||||||
|
let
|
||||||
|
|
||||||
|
cabalProject = ''
|
||||||
|
packages: .
|
||||||
|
allow-newer: aeson:*
|
||||||
|
'';
|
||||||
|
|
||||||
|
src = testSrc "cabal-simple";
|
||||||
|
|
||||||
|
# each derivation is content addressed
|
||||||
|
projectA = project' {
|
||||||
|
inherit compiler-nix-name src cabalProject;
|
||||||
|
modules = [{ contentAddressed = true; }];
|
||||||
|
};
|
||||||
|
|
||||||
|
# each derivation but one (the executable) is content addressed
|
||||||
|
projectB = project' {
|
||||||
|
inherit compiler-nix-name src cabalProject;
|
||||||
|
modules = [{
|
||||||
|
contentAddressed = true;
|
||||||
|
packages.cabal-simple.components.exes.cabal-simple.contentAddressed = false;
|
||||||
|
}];
|
||||||
|
};
|
||||||
|
|
||||||
|
exeA = projectA.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
|
||||||
|
exeB = projectB.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
|
||||||
|
|
||||||
|
in
|
||||||
|
recurseIntoAttrs {
|
||||||
|
|
||||||
|
meta.disabled = !CADerivationsEnabled;
|
||||||
|
|
||||||
|
# check if the built executables are different (one is content addressed)
|
||||||
|
# the other components are all content addressed (same output paths then)
|
||||||
|
run = stdenv.mkDerivation {
|
||||||
|
name = "ca-derivations-include-test";
|
||||||
|
|
||||||
|
buildCommand = ''
|
||||||
|
[ "${exeA}" == "${exeB}" ] && exit 1
|
||||||
|
touch $out
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta.platforms = platforms.all;
|
||||||
|
|
||||||
|
passthru = {
|
||||||
|
# Used for debugging with nix repl
|
||||||
|
inherit projectB projectA;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
59
test/ca-derivations/default.nix
Normal file
59
test/ca-derivations/default.nix
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
# Test if derivations are content addressed building two derivations producing
|
||||||
|
# the same outputs and checking if the path stores are equals
|
||||||
|
{ stdenv, pkgs, lib, mkCabalProjectPkgSet, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name, CADerivationsEnabled }:
|
||||||
|
|
||||||
|
with lib;
|
||||||
|
|
||||||
|
let
|
||||||
|
|
||||||
|
cabalProject = ''
|
||||||
|
packages: .
|
||||||
|
allow-newer: aeson:*
|
||||||
|
'';
|
||||||
|
|
||||||
|
srcPlain = testSrc "cabal-simple";
|
||||||
|
|
||||||
|
# we alter the source adding an Haskell comment since they are ignored by ghc
|
||||||
|
srcWithComment = pkgs.runCommand "src-with-comment" { } ''
|
||||||
|
mkdir $out
|
||||||
|
install ${srcPlain}/* $out
|
||||||
|
echo " -- Altering source without altering executable..." >> $out/Main.hs
|
||||||
|
'';
|
||||||
|
|
||||||
|
projectPlain = project' {
|
||||||
|
inherit compiler-nix-name cabalProject;
|
||||||
|
src = srcPlain;
|
||||||
|
modules = [{ contentAddressed = true; }];
|
||||||
|
};
|
||||||
|
|
||||||
|
projectWithComment = project' {
|
||||||
|
inherit compiler-nix-name cabalProject;
|
||||||
|
src = srcWithComment;
|
||||||
|
modules = [{ contentAddressed = true; }];
|
||||||
|
};
|
||||||
|
|
||||||
|
exe-plain = projectPlain.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
|
||||||
|
exe-withComment = projectWithComment.hsPkgs.cabal-simple.components.exes.cabal-simple.exePath;
|
||||||
|
|
||||||
|
in
|
||||||
|
recurseIntoAttrs {
|
||||||
|
|
||||||
|
meta.disabled = !CADerivationsEnabled;
|
||||||
|
|
||||||
|
run = stdenv.mkDerivation {
|
||||||
|
name = "ca-derivations-test";
|
||||||
|
|
||||||
|
buildCommand = ''
|
||||||
|
[ "${exe-plain}" != "${exe-withComment}" ] && exit 1
|
||||||
|
touch $out
|
||||||
|
'';
|
||||||
|
|
||||||
|
meta.platforms = platforms.all;
|
||||||
|
|
||||||
|
passthru = {
|
||||||
|
# Used for debugging with nix repl
|
||||||
|
inherit projectWithComment projectPlain;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
@ -4,6 +4,7 @@
|
|||||||
, nixpkgsArgs ? haskellNix.nixpkgsArgs
|
, nixpkgsArgs ? haskellNix.nixpkgsArgs
|
||||||
, ifdLevel ? 1000
|
, ifdLevel ? 1000
|
||||||
, compiler-nix-name
|
, compiler-nix-name
|
||||||
|
, CADerivationsEnabled ? false
|
||||||
, checkMaterialization ? false
|
, checkMaterialization ? false
|
||||||
}:
|
}:
|
||||||
|
|
||||||
@ -206,6 +207,8 @@ let
|
|||||||
external-static-plugin = callTest ./external-static-plugin { inherit compiler-nix-name; };
|
external-static-plugin = callTest ./external-static-plugin { inherit compiler-nix-name; };
|
||||||
exe-dlls = callTest ./exe-dlls { inherit util compiler-nix-name; };
|
exe-dlls = callTest ./exe-dlls { inherit util compiler-nix-name; };
|
||||||
exe-lib-dlls = callTest ./exe-lib-dlls { inherit util compiler-nix-name; };
|
exe-lib-dlls = callTest ./exe-lib-dlls { inherit util compiler-nix-name; };
|
||||||
|
ca-derivations = callTest ./ca-derivations { inherit compiler-nix-name CADerivationsEnabled; };
|
||||||
|
ca-derivations-include = callTest ./ca-derivations-include { inherit compiler-nix-name CADerivationsEnabled; };
|
||||||
|
|
||||||
unit = unitTests;
|
unit = unitTests;
|
||||||
};
|
};
|
||||||
|
@ -3,6 +3,8 @@
|
|||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
|
# check if Nix has the `ca-derivations` experimental features (code 0) is enabled
|
||||||
|
NIX_CA_DERIVATIONS=$(jq -e '."experimental-features".value|any(. == 0)' <<< $(nix show-config --json)) || true
|
||||||
NIX_BUILD_ARGS="${NIX_BUILD_ARGS:-}"
|
NIX_BUILD_ARGS="${NIX_BUILD_ARGS:-}"
|
||||||
|
|
||||||
cd $(dirname $0)
|
cd $(dirname $0)
|
||||||
@ -24,7 +26,8 @@ nix build $NIX_BUILD_ARGS \
|
|||||||
--option restrict-eval true \
|
--option restrict-eval true \
|
||||||
--option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
|
--option allowed-uris "https://github.com/NixOS https://github.com/input-output-hk" \
|
||||||
--no-link --keep-going -f default.nix \
|
--no-link --keep-going -f default.nix \
|
||||||
--argstr compiler-nix-name $GHC
|
--argstr compiler-nix-name $GHC \
|
||||||
|
--arg CADerivationsEnabled $NIX_CA_DERIVATIONS
|
||||||
echo >& 2
|
echo >& 2
|
||||||
|
|
||||||
printf "*** Running the unit tests... " >& 2
|
printf "*** Running the unit tests... " >& 2
|
||||||
|
Loading…
Reference in New Issue
Block a user