Split haddock out into its own derivation (#725)

Generation the Haddock documentation for a component in a dedicated
derivation (instead of as part of the derivation to build the component).
This means that the haddock documents will not be built until they are
required for instance in a shellFor shell.

Co-authored-by: Richard Wallace <richard.wallace@simspace.com>
This commit is contained in:
Hamish Mackenzie 2020-06-27 01:29:41 +12:00 committed by GitHub
parent a2033f4d02
commit 1e3c1304c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 375 additions and 85 deletions

View File

@ -1,4 +1,4 @@
{ stdenv, buildPackages, ghc, lib, gobject-introspection ? null, haskellLib, makeConfigFiles, ghcForComponent, hsPkgs, runCommand, libffi, gmp, zlib, ncurses, numactl, nodejs }:
{ stdenv, buildPackages, ghc, lib, gobject-introspection ? null, haskellLib, makeConfigFiles, haddockBuilder, ghcForComponent, hsPkgs, runCommand, libffi, gmp, zlib, ncurses, numactl, nodejs }:
lib.makeOverridable (
let self =
{ componentId
@ -43,6 +43,7 @@ let self =
, doHaddock ? component.doHaddock # Enable haddock and hoogle generation
, doHoogle ? component.doHoogle # Also build a hoogle index
, hyperlinkSource ? component.doHyperlinkSource # Link documentation to the source code
, quickjump ? component.doQuickjump # Generate an index for interactive documentation navigation
, keepSource ? component.keepSource # Build from `source` output in the store then delete `dist`
, setupHaddockFlags ? component.setupHaddockFlags
@ -70,10 +71,11 @@ let
fullName = "${nameOnly}-${package.identifier.version}";
needsProfiling = enableExecutableProfiling || enableLibraryProfiling;
configFiles = makeConfigFiles {
inherit (package) identifier;
inherit component fullName flags;
needsProfiling = enableExecutableProfiling || enableLibraryProfiling;
inherit component fullName flags needsProfiling;
};
enableFeature = enable: feature:
@ -85,6 +87,9 @@ let
[ "--prefix=$out"
"${haskellLib.componentTarget componentId}"
"$(cat ${configFiles}/configure-flags)"
] ++ commonConfigureFlags);
commonConfigureFlags = ([
# GHC
"--with-ghc=${ghc.targetPrefix}ghc"
"--with-ghc-pkg=${ghc.targetPrefix}ghc-pkg"
@ -123,7 +128,6 @@ let
"--ghc-option=-optl=-pthread"
"--ghc-option=-optl=-static"
] ++ lib.optional enableSeparateDataOutput "--datadir=$data/share/${ghc.name}"
++ lib.optional doHaddock' "--docdir=${docdir "$doc"}"
++ lib.optional (enableLibraryProfiling || enableExecutableProfiling) "--profiling-detail=${profilingDetail}"
++ lib.optional stdenv.hostPlatform.isLinux (enableFeature enableDeadCodeElimination "split-sections")
++ lib.optionals haskellLib.isCrossHost (
@ -154,26 +158,63 @@ let
if builtins.isFunction shellHook then shellHook { inherit package shellWrappers; }
else abort "shellHook should be a string or a function";
# the target dir for haddock documentation
docdir = docoutput: docoutput + "/share/doc/" + componentId.cname;
doHaddock' = doHaddock
&& (haskellLib.isLibrary componentId)
&& !haskellLib.isCrossHost;
exeExt = if stdenv.hostPlatform.isGhcjs then ".jsexe/all.js" else
if stdenv.hostPlatform.isWindows then ".exe" else "";
exeName = componentId.cname + exeExt;
testExecutable = "dist/build/${componentId.cname}/${exeName}";
in stdenv.lib.fix (drv:
# Attributes that are common to both the build and haddock derivations
commonAttrs = {
src = cleanSrc;
stdenv.mkDerivation ({
LANG = "en_US.UTF-8"; # GHC needs the locale configured during the Haddock phase.
LC_ALL = "en_US.UTF-8";
enableParallelBuilding = true;
SETUP_HS = setup + /bin/Setup;
prePatch = if (cabalFile != null)
then ''cat ${cabalFile} > ${package.identifier.name}.cabal''
else lib.optionalString (cabal-generator == "hpack") ''
${buildPackages.haskell-nix.haskellPackages.hpack.components.exes.hpack}/bin/hpack
'';
}
# patches can (if they like) depend on the version and revision of the package.
// lib.optionalAttrs (patches != []) {
patches = map (p:
if builtins.isFunction p
then p { inherit (package.identifier) version; inherit revision; }
else p
) patches;
}
// haskellLib.optionalHooks {
# These are hooks are needed to set up the source for building and running haddock
inherit preUnpack postUnpack preConfigure postConfigure;
}
// lib.optionalAttrs (stdenv.buildPlatform.libc == "glibc") {
LOCALE_ARCHIVE = "${buildPackages.glibcLocales}/lib/locale/locale-archive";
};
in
stdenv.lib.fix (drv:
let
haddock = haddockBuilder {
inherit componentId component package flags commonConfigureFlags
commonAttrs revision setupGhcOptions doHaddock
doHoogle hyperlinkSource quickjump setupHaddockFlags
needsProfiling configFiles preHaddock postHaddock;
componentDrv = drv;
};
in
stdenv.mkDerivation (commonAttrs // {
pname = nameOnly;
inherit (package.identifier) version;
src = cleanSrc;
doCheck = false;
doCrossCheck = false;
@ -184,12 +225,15 @@ stdenv.mkDerivation ({
config = component;
inherit configFiles executableToolDepends cleanSrc exeName;
env = shellWrappers;
# The directory containing the haddock documentation.
# `null' if no haddock documentation was built.
haddockDir = if doHaddock' then "${docdir drv.doc}/html" else null;
profiled = self (drvArgs // { enableLibraryProfiling = true; });
};
} // lib.optionalAttrs (haskellLib.isLibrary componentId) ({
inherit haddock;
inherit (haddock) haddockDir; # This is null if `doHaddock = false`
} // lib.optionalAttrs (haddock ? doc) {
# `doHaddock = false` turns the doc of haddock output off
inherit (haddock) doc;
}
);
meta = {
homepage = package.homepage;
@ -202,11 +246,6 @@ stdenv.mkDerivation ({
platforms = if platforms == null then stdenv.lib.platforms.all else platforms;
};
LANG = "en_US.UTF-8"; # GHC needs the locale configured during the Haddock phase.
LC_ALL = "en_US.UTF-8";
enableParallelBuilding = true;
propagatedBuildInputs =
frameworks # Frameworks will be needed at link time
# Not sure why pkgconfig needs to be propagatedBuildInputs but
@ -220,22 +259,10 @@ stdenv.mkDerivation ({
[shellWrappers buildPackages.removeReferencesTo]
++ executableToolDepends;
SETUP_HS = setup + /bin/Setup;
outputs = ["out" ]
++ (lib.optional enableSeparateDataOutput "data")
++ (lib.optional doHaddock' "doc")
++ (lib.optional keepSource "source");
# Phases
preInstallPhases = lib.optional doHaddock' "haddockPhase";
prePatch = if (cabalFile != null)
then ''cat ${cabalFile} > ${package.identifier.name}.cabal''
else lib.optionalString (cabal-generator == "hpack") ''
${buildPackages.haskell-nix.haskellPackages.hpack.components.exes.hpack}/bin/hpack
'';
configurePhase =
(lib.optionalString keepSource ''
cp -r . $source
@ -259,35 +286,6 @@ stdenv.mkDerivation ({
checkPhase = "notice: Tests are only executed by building the .run sub-derivation of this component.";
haddockPhase = ''
runHook preHaddock
# If we don't have any source files, no need to run haddock
[[ -n $(find . -name "*.hs" -o -name "*.lhs") ]] && {
docdir="${docdir "$doc"}"
mkdir -p "$docdir"
$SETUP_HS haddock \
"--html" \
${lib.optionalString doHoogle "--hoogle"} \
${lib.optionalString hyperlinkSource "--hyperlink-source"} \
${lib.concatStringsSep " " (setupHaddockFlags ++ setupGhcOptions)}
html="dist/doc/html/${componentId.cname}"
if [ -d "$html" ]; then
# Ensure that libraries are not pulled into the docs closure.
# As an example, the prettified source code of a
# Paths_package module will contain store paths of the library package.
for x in "$html/src/"*.html; do
remove-references-to -t $out $x
done
cp -R "$html" "$docdir"/html
fi
}
runHook postHaddock
'';
# Note: Cabal does *not* copy test executables during the `install` phase.
#
# Note 2: if a package contains multiple libs (lib + sublibs) SETUP register will generate a
@ -374,12 +372,11 @@ stdenv.mkDerivation ({
${shellHookApplied}
'';
}
# patches can (if they like) depend on the version and revision of the package.
// lib.optionalAttrs (patches != []) { patches = map (p: if builtins.isFunction p then p { inherit (package.identifier) version; inherit revision; } else p) patches; }
// haskellLib.optionalHooks {
inherit preUnpack postUnpack preConfigure postConfigure
# These are the hooks that are not needed by haddock (see commonAttrs for the ones that
# are shared with the haddock derivation)
inherit
preBuild postBuild
preInstall postInstall preHaddock postHaddock;
preInstall postInstall;
}
// lib.optionalAttrs (stdenv.buildPlatform.libc == "glibc"){ LOCALE_ARCHIVE = "${buildPackages.glibcLocales}/lib/locale/locale-archive"; }
)); in self)

View File

@ -3,7 +3,11 @@
let
# Builds a single component of a package.
comp-builder = haskellLib.weakCallPackage pkgs ./comp-builder.nix {
inherit ghc haskellLib makeConfigFiles ghcForComponent hsPkgs;
inherit ghc haskellLib makeConfigFiles haddockBuilder ghcForComponent hsPkgs;
};
haddockBuilder = haskellLib.weakCallPackage pkgs ./haddock-builder.nix {
inherit ghc ghcForComponent haskellLib makeConfigFiles nonReinstallablePkgs;
};
setup-builder = haskellLib.weakCallPackage pkgs ./setup-builder.nix {
@ -45,7 +49,7 @@ let
index-state = pkgs.haskell-nix.internalHackageIndexState;
}
}:
haskellLib.weakCallPackage pkgs nixpkgsHoogle {
haskellLib.weakCallPackage pkgs nixpkgsHoogle {
# For musl we can use haddock from the buildGHC
ghc = if stdenv.hostPlatform.isLinux && stdenv.targetPlatform.isMusl && !haskellLib.isNativeMusl
then ghc.buildGHC

144
builder/haddock-builder.nix Normal file
View File

@ -0,0 +1,144 @@
{ stdenv, buildPackages, lib, haskellLib, ghc, ghcForComponent, nonReinstallablePkgs, runCommand, writeText, writeScript, makeConfigFiles }:
{ componentId
, component
, package
, flags
, revision
, commonAttrs
, preHaddock
, postHaddock
, commonConfigureFlags
, doHaddock
, doHoogle
, hyperlinkSource
, quickjump
, setupHaddockFlags
, setupGhcOptions
, needsProfiling
, componentDrv
, configFiles # component config files
}:
let
doHaddock' = doHaddock
&& (haskellLib.isLibrary componentId)
&& !haskellLib.isCrossHost;
# the target dir for haddock documentation
docdir = docoutput: docoutput + "/share/doc/" + componentId.cname;
packageCfgDir = configFiles.packageCfgDir;
fullName = "${componentDrv.name}-haddock";
# These config files are like the one used in the build derivation,
# but `chooseDrv` will be used to map all the references to libraries
# to their haddock derivation.
docsConfigFiles = makeConfigFiles {
inherit (package) identifier;
inherit component fullName flags needsProfiling;
chooseDrv = p: p.haddock;
};
finalConfigureFlags = lib.concatStringsSep " " (
[ "--prefix=${componentDrv}"
"${haskellLib.componentTarget componentId}"
"$(cat ${docsConfigFiles}/configure-flags)"
]
++ commonConfigureFlags
++ lib.optional doHaddock' " --docdir=${docdir "$doc"}");
shellWrappers = ghcForComponent {
componentName = fullName;
configFiles = docsConfigFiles;
};
in stdenv.lib.fix (drv: stdenv.mkDerivation (commonAttrs // {
name = fullName;
passthru = {
configFiles = docsConfigFiles;
# The directory containing the haddock documentation.
haddockDir = if doHaddock' then "${docdir drv.doc}/html" else null;
};
# `out` contains the `package.conf.d` files used for building the
# haddock files.
# `doc` contains just the haddock output files.
outputs = ["out"]
++ lib.optional doHaddock' "doc";
nativeBuildInputs =
[ shellWrappers buildPackages.removeReferencesTo ]
++ componentDrv.executableToolDepends;
configurePhase = ''
echo Configure flags:
printf "%q " ${finalConfigureFlags}
echo
$SETUP_HS configure ${finalConfigureFlags}
'';
buildPhase = ''
mkdir -p $out
'' + lib.optionalString doHaddock' ''
runHook preHaddock
# If we don't have any source files, no need to run haddock
[[ -n $(find . -name "*.hs" -o -name "*.lhs") ]] && {
$SETUP_HS haddock \
"--html" \
${lib.optionalString doHoogle "--hoogle"} \
${lib.optionalString hyperlinkSource "--hyperlink-source"} \
${lib.optionalString quickjump "--quickjump"} \
${lib.concatStringsSep " " (setupHaddockFlags ++ setupGhcOptions)}
}
runHook postHaddock
'';
installPhase =
let
target-pkg-and-db = "${ghc.targetPrefix}ghc-pkg -v0 --package-db $out/package.conf.d";
in ''
html="dist/doc/html/${package.identifier.name}"
if [ -d "$html" ]; then
# Ensure that libraries are not pulled into the docs closure.
# As an example, the prettified source code of a
# Paths_package module will contain store paths of the library package.
for x in "$html/src/"*.html; do
remove-references-to -t $out $x
remove-references-to -t ${componentDrv} $x
done
docdir="${docdir "$doc"}"
mkdir -p "$docdir"
cp -R "$html" "$docdir"/html
fi
${ghc.targetPrefix}ghc-pkg -v0 init $out/package.conf.d
for i in "${componentDrv}/package.conf.d"/*.conf; do
# Copy the .conf files from the build derivation, but replace the `haddock-intefaces:`
# field with correct location for haddocks (now it is known). This will insure that
# the other haddock derivations build to reference this one will have the correct
# working hyper links.
pkg=$(basename "$i")
sed -e "s|haddock-interfaces:.*|haddock-interfaces: $docdir/html/${componentId.cname}.haddock|" -e "s|haddock-html:.*|haddock-html: $docdir/html/|" "$i" > "$pkg"
${ghc.targetPrefix}ghc-pkg -v0 --package-db ${docsConfigFiles}/${configFiles.packageCfgDir} -f $out/package.conf.d register "$pkg"
done
ln -s ${componentDrv}/exactDep $out/exactDep
ln -s ${componentDrv}/envDep $out/envDep
'';
}
// haskellLib.optionalHooks {
inherit preHaddock postHaddock;
}
))

View File

@ -1,5 +1,7 @@
{ stdenv, lib, haskellLib, ghc, nonReinstallablePkgs, runCommand, writeText, writeScript }:
{ identifier, component, fullName, flags ? {}, needsProfiling ? false, chooseDrv ? drv: drv }:
let
flagsAndConfig = field: xs: lib.optionalString (xs != []) ''
echo ${lib.concatStringsSep " " (map (x: "--${field}=${x}") xs)} >> $out/configure-flags
@ -41,20 +43,21 @@ let
libDir = "lib/${ghcCommand}-${ghc.version}";
packageCfgDir = "${libDir}/package.conf.d";
in { identifier, component, fullName, flags ? {}, needsProfiling ? false }:
# Filters out only library packages that for this GHC target
# TODO investigate why this is needed
# TODO find out why p ? configFiles helps (for instance for `R1909.aarch64-unknown-linux-gnu.tests.cabal-22.run.x86_64-linux`)
let libDeps = (if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
(lib.filter (p: (p ? configFiles) && p.configFiles.targetPrefix == ghc.targetPrefix)
(map getLibComponent component.depends));
cfgFiles =
let xs = map
(p: "${p.configFiles}")
libDeps;
in lib.concatStringsSep "\" \"" xs;
libs = lib.concatMapStringsSep "\" \"" (p: "${p}") libDeps;
in
libDeps = map chooseDrv (
(if needsProfiling then (x: map (p: p.profiled or p) x) else x: x)
(lib.filter (p: (p ? configFiles) && p.configFiles.targetPrefix == ghc.targetPrefix)
(map getLibComponent component.depends))
);
cfgFiles =
let xs = map
(p: "${p.configFiles}")
libDeps;
in lib.concatStringsSep "\" \"" xs;
libs = lib.concatMapStringsSep "\" \"" (p: "${p}") libDeps;
in
runCommand "${ghc.targetPrefix}${fullName}-config" {
nativeBuildInputs = [ghc];
passthru = {

View File

@ -1,6 +1,12 @@
This file contains a summary of changes to Haskell.nix and `nix-tools`
that will impact users.
## June 25, 2019
* Haddock docs are now built in their own derivation when needed (not as part
of the component build).
They should build automatically when something (such as `shellFor`) attempts
to accesses the `.doc` attribute of component.
## December 27, 2019
* Fix overlays/bootstrap.nix to provide LLVM 6, not LLVM 5, to ghc-8.6.X compilers.

View File

@ -101,7 +101,12 @@ let
doHyperlinkSource = mkOption {
description = "Link documentation to the source code.";
type = bool;
default = (def.doHoogle or true);
default = (def.doHyperlinkSource or true);
};
doQuickjump = mkOption {
description = "Generate an index for interactive documentation navigation.";
type = bool;
default = (def.doQuickjump or true);
};
dontPatchELF = mkOption {
description = "If set, the patchelf command is not used to remove unnecessary RPATH entries. Only applies to Linux.";

View File

@ -187,6 +187,9 @@ let
# This test makes a plan for building cabal 3.2 using index-states that will
# never work with ghc 8.10.1
index-state = callTest ./index-state {};
} // lib.optionalAttrs (!pkgs.haskell-nix.haskellLib.isCrossHost) {
# Haddock is not included with cross compilers currently
sublib-docs = callTest ./sublib-docs { inherit util; };
};
# This is the same as allTests, but filter out all the key/vaules from the

View File

@ -0,0 +1,2 @@
import Distribution.Simple
main = defaultMain

View File

@ -0,0 +1,63 @@
# Test a package set
{ stdenv, util, cabalProject', haskellLib, recurseIntoAttrs, testSrc }:
with stdenv.lib;
let
project = cabalProject' {
src = testSrc "sublib-docs";
};
packages = project.hsPkgs;
in recurseIntoAttrs {
ifdInputs = {
inherit (project) plan-nix;
};
run = stdenv.mkDerivation {
name = "sublib-docs-test";
buildCommand = ''
exe="${packages.sublib-docs.components.exes.sublib-docs}/bin/sublib-docs${stdenv.hostPlatform.extensions.executable}"
size=$(command stat --format '%s' "$exe")
printf "size of executable $exe is $size. \n" >& 2
# fixme: run on target platform when cross-compiled
printf "checking whether executable runs... " >& 2
cat ${haskellLib.check packages.sublib-docs.components.exes.sublib-docs}
'' +
# Musl and Aarch are statically linked..
optionalString (!stdenv.hostPlatform.isAarch32 && !stdenv.hostPlatform.isAarch64 && !stdenv.hostPlatform.isMusl) (''
printf "checking that executable is dynamically linked to system libraries... " >& 2
'' + optionalString (stdenv.isLinux && !stdenv.hostPlatform.isMusl) ''
ldd $exe | grep libgmp
'' + optionalString stdenv.isDarwin ''
otool -L $exe |grep .dylib
'') + ''
printf "Checking that \"all\" component has the programs... " >& 2
all_exe="${packages.sublib-docs.components.all}/bin/sublib-docs${stdenv.hostPlatform.extensions.executable}"
test -f "$all_exe"
echo "$all_exe" >& 2
# Check that it looks like we have docs
test -f "${packages.sublib-docs.components.library.doc}/share/doc/sublib-docs/html/Lib.html"
test -f "${packages.sublib-docs.components.sublibs.slib.doc}/share/doc/slib/html/Slib.html"
touch $out
'';
meta.platforms = platforms.all;
passthru = {
# Used for debugging with nix repl
inherit packages;
# Used for testing externally with nix-shell (../tests.sh).
# This just adds cabal-install to the existing shells.
test-shell = util.addCabalInstall packages.sublib-docs.components.all;
};
};
}

View File

@ -0,0 +1,6 @@
-- | Lib with haddocks
module Lib where
-- | world string
world :: String
world = "world"

View File

@ -0,0 +1,6 @@
-- | sublib with haddocks
module Slib where
-- | hello string
hello :: String
hello = "Hello"

View File

@ -0,0 +1,4 @@
module Main where
main :: IO ()
main = putStrLn "Hello, Haskell!"

View File

@ -0,0 +1,47 @@
cabal-version: 2.2
-- Initial package description 'cabal-simple.cabal' generated by 'cabal
-- init'. For further documentation, see
-- http://haskell.org/cabal/users-guide/
name: sublib-docs
version: 0.1.0.0
-- synopsis:
-- description:
-- bug-reports:
license: MIT
author: Moritz Angermann
maintainer: moritz.angermann@iohk.io
-- category:
build-type: Simple
library
exposed-modules: Lib
-- other-modules:
-- other-extensions:
build-depends: base
, slib
-- hs-source-dirs:
default-language: Haskell2010
hs-source-dirs: lib
library slib
exposed-modules: Slib
build-depends: extra
, safe
, aeson
, base
default-language: Haskell2010
hs-source-dirs: slib
executable sublib-docs
main-is: Main.hs
-- other-modules:
-- other-extensions:
build-depends: base
, sublib-docs
, extra
, optparse-applicative
hs-source-dirs: src
-- hs-source-dirs:
default-language: Haskell2010