Add getComponent to project and package (#1060)

* Delays cabal configure errors until after evaluation when possible.

* Avoids rerunning long `cabal configure` if we already know it will fail.

* Provides a way to get a component from a package or project using a `cabal` like reference.

Code using the `tool` functions will automatically use `getComponent`.

For `(hackage-package {...}).components.library` is also ok.

```
# Consider changing hackage-package use:
(pkgs.haskell-nix.hackage-package {...}).components.exes.something
(pkgs.haskell-nix.hackage-package {...}).getComponent "exe:something"

# For any cabal project:
project.hsPkgs.somepackage.components.exes.something
project.getComponent "somepackage:exe:something"
# or do it in two steps
(project.getPackage "somepackage").getComponent "exe:something"
```

The reason for the new function is that we cannot provide the attribute interface without knowing that packages are in the project first.

Here is how the `cabal configure` error output is handled:

* The `plan-nix` derivation builds even if `cabal configure` fails.

* When it fails, it copies `failed-cabal-configure.nix` to the `$out/default.nix` along with a copy of the `cabal configure` output.

* When `failed-cabal-configure.nix` is imported and used in any way it writes the `cabal configure` output with `__trace` so it will always be visible.

* Instead of a `plan` the imported nix contains a `configurationError` pointing the `cabal configure` output.

* The intermediate functions `configurationError` and bubble it up to where it is needed.

* `getPackage` returns a mostly empty proxy for a real package when there is a `configurationError`

* The `getComponent` function always returns a derivation, but the version in the proxy writes the `cabal configure` output to stdout and calls `exit 1` (so that it will never build).
This commit is contained in:
Hamish Mackenzie 2021-03-04 20:10:58 +13:00 committed by GitHub
parent cb264f6f6a
commit 5c25ce919f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 175 additions and 53 deletions

10
ci.nix
View File

@ -80,9 +80,9 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
} // pkgs.lib.optionalAttrs runTests {
inherit (build) tests tools maintainer-scripts maintainer-script-cache;
} // pkgs.lib.optionalAttrs (ifdLevel >= 1) {
iserv-proxy = pkgs.ghc-extra-packages."${compiler-nix-name}".iserv-proxy.components.exes.iserv-proxy;
iserv-proxy = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "iserv-proxy:exe:iserv-proxy";
} // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).getComponent "exe:hello";
});
}
//
@ -102,10 +102,10 @@ dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
inherit (build) tests;
}) // pkgs.lib.optionalAttrs (ifdLevel >= 2 && crossSystemName != "ghcjs") {
# GHCJS builds its own template haskell runner.
remote-iserv = pkgs.ghc-extra-packages."${compiler-nix-name}".remote-iserv.components.exes.remote-iserv;
iserv-proxy = pkgs.ghc-extra-packages."${compiler-nix-name}".iserv-proxy.components.exes.iserv-proxy;
remote-iserv = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "remote-iserv:exe:remote-iserv";
iserv-proxy = pkgs.ghc-extra-projects."${compiler-nix-name}".getComponent "iserv-proxy:exe:iserv-proxy";
} // pkgs.lib.optionalAttrs (ifdLevel >= 3) {
hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).components.exes.hello;
hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; inherit compiler-nix-name; }).getComponent "exe:hello";
})
))
)

View File

@ -24,21 +24,21 @@ let
all-ghcjs = pkgs.buildPackages.symlinkJoin {
name = "ghcjs-${ghcjsVersion}-symlinked";
paths = [
ghcjs.components.exes.ghcjs
ghcjs.components.exes.ghcjs-pkg
ghcjs.components.exes.ghcjs-boot
ghcjs.components.exes.ghcjs-dumparchive
(ghcjs.getComponent "exe:ghcjs")
(ghcjs.getComponent "exe:ghcjs-pkg")
(ghcjs.getComponent "exe:ghcjs-boot")
(ghcjs.getComponent "exe:ghcjs-dumparchive")
] ++ (if isGhcjs88
then [
ghcjs.components.exes.haddock
ghcjs.components.exes.private-ghcjs-run
ghcjs.components.exes.private-ghcjs-unlit
ghcjs.components.exes.private-ghcjs-hsc2hs
(ghcjs.getComponent "exe:haddock")
(ghcjs.getComponent "exe:private-ghcjs-run")
(ghcjs.getComponent "exe:private-ghcjs-unlit")
(ghcjs.getComponent "exe:private-ghcjs-hsc2hs")
]
else [
ghcjs.components.exes.haddock-ghcjs
ghcjs.components.exes.hsc2hs-ghcjs
ghcjs.components.exes.ghcjs-run
(ghcjs.getComponent "exe:haddock-ghcjs")
(ghcjs.getComponent "exe:hsc2hs-ghcjs")
(ghcjs.getComponent "exe:ghcjs-run")
]);
};
libexec = "libexec/${builtins.replaceStrings ["darwin" "i686"] ["osx" "i386"] pkgs.stdenv.buildPlatform.system}-${ghc.name}/ghcjs-${ghcVersion}";
@ -68,9 +68,9 @@ let
lndir ${all-ghcjs}/bin $out/bin
chmod -R +w $out/bin
rm $out/bin/ghcjs-boot
cp ${ghcjs.components.exes.ghcjs-boot}/bin/ghcjs-boot $out/bin
cp ${ghcjs.getComponent "exe:ghcjs-boot"}/bin/ghcjs-boot $out/bin
rm $out/bin/haddock
cp ${ghcjs.components.exes.haddock}/bin/haddock $out/bin
cp ${ghcjs.getComponent "exe:haddock"}/bin/haddock $out/bin
wrapProgram $out/bin/ghcjs --add-flags "-B$out/lib"
wrapProgram $out/bin/haddock --add-flags "-B$out/lib"

View File

@ -44,6 +44,7 @@ in
let
forName = pkgs.lib.optionalString (name != null) (" for " + name);
nameAndSuffix = suffix: if name == null then suffix else name + "-" + suffix;
ghc' =
if ghcOverride != null
@ -378,7 +379,7 @@ let
else null;
} // pkgs.lib.optionalAttrs (checkMaterialization != null) {
inherit checkMaterialization;
}) (pkgs.evalPackages.runCommand (if name == null then "plan-to-nix-pkgs" else name + "-plan-to-nix-pkgs") {
}) (pkgs.evalPackages.runCommand (nameAndSuffix "plan-to-nix-pkgs") {
nativeBuildInputs = [ nix-tools dummy-ghc dummy-ghc-pkg cabal-install pkgs.evalPackages.rsync ];
# Needed or stack-to-nix will die on unicode inputs
LOCALE_ARCHIVE = pkgs.lib.optionalString (pkgs.evalPackages.stdenv.buildPlatform.libc == "glibc") "${pkgs.evalPackages.glibcLocales}/lib/locale/locale-archive";
@ -386,10 +387,32 @@ let
meta.platforms = pkgs.lib.platforms.all;
preferLocalBuild = false;
outputs = [
"out" # The results of plan-to-nix
"json" # The `plan.json` file generated by cabal and used for `plan-to-nix` input
"freeze" # The `cabal.project.freeze` file created by `cabal v2-freeze`
"out" # The results of plan-to-nix
# These two output will be present if in cabal configure failed.
# They are used to provide passthru.json and passthru.freeze that
# check first for cabal configure failure.
"maybeJson" # The `plan.json` file generated by cabal and used for `plan-to-nix` input
"maybeFreeze" # The `cabal.project.freeze` file created by `cabal v2-freeze`
];
passthru =
let
checkCabalConfigure = ''
if [[ -f ${plan-nix}/cabal-configure.out ]]; then
cat ${plan-nix}/cabal-configure.out
exit 1
fi
'';
in {
# These check for cabal configure failure
json = pkgs.evalPackages.runCommand (nameAndSuffix "plan-json") {} ''
${checkCabalConfigure}
cp ${plan-nix.maybeJson} $out
'';
freeze = pkgs.evalPackages.runCommand (nameAndSuffix "plan-freeze") {} ''
${checkCabalConfigure}
cp ${plan-nix.maybeFreeze} $out
'';
};
} ''
tmp=$(mktemp -d)
cd $tmp
@ -430,12 +453,14 @@ let
export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
export GIT_SSL_CAINFO=${cacert}/etc/ssl/certs/ca-bundle.crt
mkdir -p $out
# Using `cabal v2-freeze` will configure the project (since
# it is not configured yet), taking the existing `cabal.project.freeze`
# file into account. Then it "writes out a freeze file which
# records all of the versions and flags that are picked" (from cabal docs).
echo "Using index-state ${index-state-found}"
HOME=${
if(HOME=${
# This creates `.cabal` directory that is as it would have
# been at the time `cached-index-state`. We may include
# some packages that will be excluded by `index-state-found`
@ -460,14 +485,12 @@ let
--enable-benchmarks \
${pkgs.lib.optionalString (ghc.targetPrefix == "js-unknown-ghcjs-")
"--ghcjs --with-ghcjs=js-unknown-ghcjs-ghc --with-ghcjs-pkg=js-unknown-ghcjs-ghc-pkg"} \
${configureArgs}
${configureArgs} 2>&1 | tee -a cabal-configure.out); then
cp cabal.project.freeze $freeze
cp cabal.project.freeze $maybeFreeze
# Not needed any more (we don't want it to wind up in the $out hash)
rm cabal.project.freeze
mkdir -p $out
# ensure we have all our .cabal files (also those generated from package.yaml) files.
# otherwise we'd need to be careful about putting the `cabal-generator = hpack` into
# the nix expression. As we already called `hpack` on all `package.yaml` files we can
@ -489,7 +512,7 @@ let
(cd $out${subDir'} && plan-to-nix --full --plan-json $tmp${subDir'}/dist-newstyle/cache/plan.json -o .)
# Make the plan.json file available in case we need to debug plan-to-nix
cp $tmp${subDir'}/dist-newstyle/cache/plan.json $json
cp $tmp${subDir'}/dist-newstyle/cache/plan.json $maybeJson
# Remove the non nix files ".project" ".cabal" "package.yaml" files
# as they should not be in the output hash (they may change slightly
@ -504,6 +527,18 @@ let
# move pkgs.nix to default.nix ensure we can just nix `import` the result.
mv $out${subDir'}/pkgs.nix $out${subDir'}/default.nix
else
# When cabal configure fails copy the output that we captured above and
# use `failed-cabal-configure.nix` to make a suitable derviation with.
cp cabal-configure.out $out
cp ${./failed-cabal-configure.nix} $out/default.nix
# These should only be used indirectly by `passthru.json` and `passthru.freeze`.
# Those derivations will check for `cabal-configure.out` out first to see if
# it is ok to use these files.
echo "Cabal configure failed see $out/cabal-configure.out for details" > $maybeJson
echo "Cabal configure failed see $out/cabal-configure.out for details" > $maybeFreeze
fi
'');
in {
projectNix = plan-nix;

View File

@ -62,6 +62,12 @@ in {
benchmarks = "bench";
};
# For looking up the components attribute based on the cabal component type
prefixComponent =
lib.listToAttrs (
lib.mapAttrsToList (value: name: { inherit name value; })
componentPrefix);
applyComponents = f: config:
let
comps = config.components;

View File

@ -0,0 +1,9 @@
let
configurationError = ./cabal-configure.out;
in
# Trace the error output to make sure the user has a chance to see it
# (even if the choose not to build anything from the project)
__trace ''
ERROR: cabal configure failed with:
${__readFile configurationError}
'' { inherit configurationError; }

View File

@ -581,7 +581,7 @@ in {
cabal-install = final.evalPackages.haskell-nix.cabal-install-unchecked.${compiler-nix-name};
nix-tools = final.evalPackages.haskell-nix.nix-tools-unchecked.${compiler-nix-name};
materialized = ../materialized + "/${compiler-nix-name}/cabal-install";
} // args)).components.exes.cabal;
} // args)).getComponent "exe:cabal";
nix-tools-set = { compiler-nix-name, ... }@args:
let
project =
@ -617,7 +617,22 @@ in {
];
}];
} // args);
exes = project.nix-tools.components.exes // project.hpack.components.exes;
exes =
let
package = project.getPackage "nix-tools";
in (builtins.map (name: package.getComponent "exe:${name}") [
"cabal-to-nix"
"hashes-to-nix"
"plan-to-nix"
"hackage-to-nix"
"lts-to-nix"
"stack-to-nix"
"truncate-index"
"stack-repos"
"cabal-name"
]) ++ [
(project.getComponent "hpack:exe:hpack")
];
tools = [
final.buildPackages.nix
# Double buildPackages is intentional, see comment in lib/default.nix for details.
@ -626,7 +641,7 @@ in {
in
final.symlinkJoin {
name = "nix-tools";
paths = builtins.attrValues exes;
paths = exes;
buildInputs = [ final.makeWrapper ];
meta.platforms = final.lib.platforms.all;
# We wrap the -to-nix executables with the executables from `tools` (e.g. nix-prefetch-git)
@ -774,7 +789,7 @@ in {
version = "1.24.4";
inherit ghcOverride nix-tools cabal-install index-state;
materialized = ../materialized/bootstrap + "/${buildBootstrapper.compilerNixName}/hscolour";
} // args)).components.exes.HsColour;
} // args)).getComponent "exe:HsColour";
hscolour = bootstrap.packages.hscolour-tool {};
hscolour-unchecked = bootstrap.packages.hscolour-tool { checkMaterialization = false; };
};

View File

@ -457,7 +457,7 @@ final: prev: {
hackage-package =
{ name, compiler-nix-name, ... }@args':
let args = { caller = "hackage-package"; } // args';
in (hackage-project args).hsPkgs.${name};
in (hackage-project args).getPackage name;
hackage-project =
{ name
, compiler-nix-name
@ -500,11 +500,19 @@ final: prev: {
let
args = { caller = "cabalProject'"; } // args';
callProjectResults = callCabalProjectToNix args;
in let pkg-set = mkCabalProjectPkgSet
{ inherit compiler-nix-name;
plan-pkgs = importAndFilterProject {
inherit (callProjectResults) projectNix sourceRepos src;
plan-pkgs = importAndFilterProject {
inherit (callProjectResults) projectNix sourceRepos src;
};
pkg-set = if plan-pkgs ? configurationError
then {
inherit (plan-pkgs) configurationError;
config = {
compiler.nix-name = compiler-nix-name;
hsPkgs = {};
};
}
else mkCabalProjectPkgSet
{ inherit compiler-nix-name plan-pkgs;
pkg-def-extras = args.pkg-def-extras or [];
modules = (args.modules or [])
++ final.lib.optional (args ? ghcOverride || args ? ghc)
@ -514,7 +522,7 @@ final: prev: {
extra-hackages = args.extra-hackages or [];
};
project = addProjectAndPackageAttrs rec {
project = addProjectAndPackageAttrs rec {
inherit (pkg-set.config) hsPkgs;
inherit pkg-set;
plan-nix = callProjectResults.projectNix;
@ -535,7 +543,7 @@ final: prev: {
final.lib.fix (project':
let project = project' // { recurseForDerivations = false; };
in rawProject // rec {
hsPkgs = final.lib.mapAttrs (n: package':
hsPkgs = final.lib.mapAttrs (packageName: package':
if package' == null
then null
else
@ -548,6 +556,16 @@ final: prev: {
) package'.components;
inherit project;
# Look up a component in the package based on ctype:name
getComponent = componentName:
let m = builtins.match "(lib|flib|exe|test|bench):([^:]*)" componentName;
in
assert final.lib.asserts.assertMsg (m != null)
"Invalid package component name ${componentName}. Expected it to start with one of lib: flib: exe: test: or bench:";
if builtins.elemAt m 0 == "lib" && builtins.elemAt m 1 == packageName
then components.library
else components.${haskellLib.prefixComponent.${builtins.elemAt m 0}}.${builtins.elemAt m 1};
coverageReport = haskellLib.coverageReport (rec {
name = package.identifier.name + "-" + package.identifier.version;
library = if components ? library then components.library else null;
@ -568,6 +586,48 @@ final: prev: {
rawProject.projectFunction pkgs.haskell-nix rawProject.projectArgs
) final.pkgsCross) // { recurseForDerivations = false; };
# Like `.hsPkgs.${packageName}` but when compined with `getComponent` any
# cabal configure errors are defered until the components derivation builds.
getPackage = packageName:
if rawProject.pkg-set ? configurationError
then
# A minimal proxy for a package when cabal configure failed
let package = {
# Including the project so that things like:
# (p.getPackage "hello").project.tool "hlint" "latest"
# will still work even if "hello" failed to configure.
inherit project;
# Defer configure time errors for the library component
# (p.getPackage "hello").components.library
components.library = package.getComponent "lib:${packageName}";
# This procide a derivation (even though the component may
# not exist at all). The derivation will never build
# and simple outputs the result of cabal configure.
getComponent = componentName:
final.evalPackages.runCommand "cabal-configure-error" {
passthru = {
inherit project package;
};
} ''
cat ${rawProject.pkg-set.configurationError}
echo Unable to find component ${packageName}:${componentName} \
due to the above cabal configuration error
exit 1
'';
};
in package
else project.hsPkgs.${packageName};
# Look a component in the project based on `pkg:ctype:name`
getComponent = componentName:
let m = builtins.match "([^:]*):(lib|flib|exe|test|bench):([^:]*)" componentName;
in
assert final.lib.asserts.assertMsg (m != null)
"Invalid package component name ${componentName}. Expected package:ctype:component (where ctype is one of lib, flib, exe, test, or bench)";
(getPackage (builtins.elemAt m 0)).getComponent "${builtins.elemAt m 1}:${builtins.elemAt m 2}";
# Helper function that can be used to make a Nix Flake out of a project
# by including a flake.nix. See docs/tutorials/getting-started-flakes.md
# for an example flake.nix file.

View File

@ -66,7 +66,7 @@ in { haskell-nix = prev.haskell-nix // {
{ configureArgs = "--disable-benchmarks --disable-tests"; }
// args
// { name = final.haskell-nix.toolPackageName.${name} or name; }))
.components.exes."${final.haskell-nix.packageToolName.${name} or name}";
.getComponent "exe:${final.haskell-nix.packageToolName.${name} or name}";
tool = compiler-nix-name: name: versionOrArgs:
let

View File

@ -4,12 +4,12 @@
let
eval = (import ../../. {}).pkgs;
linux = (import ../../. { checkMaterialization = true; system = "x86_64-linux"; }).pkgs;
darwin = (import ../../. { checkMaterialization = true; system = "x86_64-darwin"; }).pkgs;
darwin = (import ../../. { checkMaterialization = true; system = "x86_64-darwin"; }).pkgs-unstable;
in eval.linkFarm "check-${compiler-nix-name}" [
# This set of derivations should be enough to ensure all the materialized files for a
# given GHC version are checked.
{ name = "linux-cabal-install"; path = linux.haskell-nix.cabal-install.${compiler-nix-name}; }
# { name = "darwin-cabal-install"; path = darwin.haskell-nix.cabal-install.${compiler-nix-name}; }
{ name = "darwin-cabal-install"; path = darwin.haskell-nix.cabal-install.${compiler-nix-name}; }
{ name = "linux-nix-tools"; path = linux.haskell-nix.nix-tools.${compiler-nix-name}; }
{ name = "linux"; path = linux.ghc-extra-projects.${compiler-nix-name}.plan-nix; }
# In some cased you may need comment out one or more of these if the GHC version needed cannot be built.

View File

@ -21,8 +21,6 @@ let
inherit modules;
};
packages = project.hsPkgs;
in recurseIntoAttrs {
ifdInputs = {
inherit (project) plan-nix;
@ -31,7 +29,7 @@ in recurseIntoAttrs {
name = "cabal-simple-prof-test";
buildCommand = ''
exe="${packages.cabal-simple.components.exes.cabal-simple.exePath}"
exe="${(project.getComponent "cabal-simple:exe:cabal-simple").exePath}"
size=$(command stat --format '%s' "$exe")
printf "size of executable $exe is $size. \n" >& 2
@ -41,7 +39,7 @@ in recurseIntoAttrs {
# Curiosity: cross compilers prodcing profiling with `+RTS -p -h` lead to the following cryptic message:
# cabal-simple: invalid heap profile option: -h*
# Hence we pass `-hc`.
${toString packages.cabal-simple.components.exes.cabal-simple.config.testWrapper} $exe +RTS -p -hc
${toString (project.getComponent "cabal-simple:exe:cabal-simple").config.testWrapper} $exe +RTS -p -hc
touch $out
'';
@ -54,7 +52,7 @@ in recurseIntoAttrs {
passthru = {
# Used for debugging with nix repl
inherit project packages;
inherit project;
};
};
}

View File

@ -5,5 +5,5 @@ in recurseIntoAttrs {
ifdInputs = {
inherit (project) plan-nix;
};
build = project.hsPkgs.haskell-language-server.components.exes.haskell-language-server;
build = project.getComponent "haskell-language-server:exe:haskell-language-server";
}

View File

@ -6,7 +6,7 @@ with lib;
let
# The hackage-security 0.6.0.1 was uploaded at 2020-04-06T20:54:35Z
# See https://hackage.haskell.org/package/hackage-security-0.6.0.1
version-used-at = index-state: (tool compiler-nix-name "cabal" {
version-used-at = index-state: ((tool compiler-nix-name "cabal" {
version = "3.2.0.0";
inherit index-state;
cabalProject = ''
@ -15,7 +15,7 @@ let
package cabal-install
flags: -native-dns
'';
}).project.hsPkgs.hackage-security.components.library.version;
}).project.getPackage "hackage-security").components.library.version;
version-before = version-used-at "2020-04-06T20:54:34Z";
version-after = version-used-at "2020-04-06T20:54:35Z";

View File

@ -15,7 +15,6 @@ let
{ reinstallableLibGhc = true; } ];
};
packages = project.hsPkgs;
meta = {
platforms = platforms.unix;
# Building reinstallable lib GHC is broken on 8.10, and we require lib ghc so this won't work with cross-compiling.
@ -32,7 +31,7 @@ recurseIntoAttrs ({
name = "setup-deps-test";
buildCommand = ''
exe="${packages.pkg.components.exes.pkg}/bin/pkg"
exe="${project.getComponent "pkg:exe:pkg"}/bin/pkg"
printf "checking whether executable runs... " >& 2
$exe
@ -43,7 +42,7 @@ recurseIntoAttrs ({
inherit meta;
passthru = {
# Attributes used for debugging with nix repl
inherit project packages;
inherit project;
};
};
})