mirror of
https://github.com/ilyakooo0/haskell.nix.git
synced 2024-08-15 18:00:50 +03:00
Improve coverage reports (#1548)
* Provide an empty coverage report when package has no coverage
- When covering a project with multiple packages, the project coverage report
will produce a table of individual coverage reports for each package. The
links in this table don't go anywhere when a package is uncovered (they are
broken links).
- By producing an empty coverage report when a package has no coverage, we fix
the broken links, and provide a tad more information about the coverage
state of that package.
* Improve project coverage report index page
- Provide a link to the union/all coverage report.
- Format the reports as a list instead of a table of one column.
- Provide better explanation of what each report means.
* Add warning explaining modules with no coverage
- Modules that have no coverage at all are simply not included in the HTML
reports generated by HPC.
- Add a warning to the project coverage report index page so users are aware of
this limitation of HPC.
* Simplify interface of coverageReport
- Remove the concept of "package boundaries" from the "coverageReport" function.
- The "coverageReport" is now a function of:
- arbitrary checks generating tix files
- arbitrary mix modules
- This more closely reflects the usage of hpc, which doesn't care about package
boundaries.
- Use this new "coverageReport" function to simplify the "projectCoverageReport"
implementation. A project coverage report now simply:
- copies out constituent coverage reports.
- writes out an "all coverage report" using all checks in the project and all
mix modules..
- writes out a summary index page.
This commit is contained in:
Samuel Evans-Powell
GitHub
parent
80082aebba
commit
0dca71e2f3
@ -10,19 +10,12 @@ coverageReports:
|
||||
let
|
||||
toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";
|
||||
|
||||
# Create table rows for a project coverage index page that look something like:
|
||||
#
|
||||
# | Package |
|
||||
# |------------------|
|
||||
# | cardano-shell |
|
||||
# | cardano-launcher |
|
||||
coverageTableRows = coverageReport:
|
||||
# Create a list element for a project coverage index page.
|
||||
coverageListElement = coverageReport:
|
||||
''
|
||||
<tr>
|
||||
<td>
|
||||
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</href>
|
||||
</td>
|
||||
</tr>
|
||||
<li>
|
||||
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</a>
|
||||
</li>
|
||||
'';
|
||||
|
||||
projectIndexHtml = pkgs.writeText "index.html" ''
|
||||
@ -31,23 +24,32 @@ let
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
||||
</head>
|
||||
<body>
|
||||
<table border="1" width="100%">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th>Report</th>
|
||||
</tr>
|
||||
|
||||
${with lib; concatStringsSep "\n" (map coverageTableRows coverageReports)}
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
<div>
|
||||
WARNING: Modules with no coverage are not included in any of these reports, this is just how HPC works under the hood.
|
||||
</div>
|
||||
<div>
|
||||
<h2>Union Report</h2>
|
||||
<p>The following report shows how each module is covered by any test in the project:</p>
|
||||
<ul>
|
||||
<li>
|
||||
<a href="all/hpc_index.html">all</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div>
|
||||
<h2>Individual Reports</h2>
|
||||
<p>The following reports show how the tests of each package cover modules in the project:</p>
|
||||
<ul>
|
||||
${with lib; concatStringsSep "\n" (map coverageListElement coverageReports)}
|
||||
</ul>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
'';
|
||||
|
||||
ghc = project.pkg-set.config.ghc.package;
|
||||
|
||||
libs = lib.remove null (map (r: r.library) coverageReports);
|
||||
libs = lib.unique (lib.concatMap (r: r.mixLibraries) coverageReports);
|
||||
|
||||
mixDirs =
|
||||
map
|
||||
@ -56,6 +58,17 @@ let
|
||||
|
||||
srcDirs = map (l: l.srcSubDirPath) libs;
|
||||
|
||||
allCoverageReport = haskellLib.coverageReport {
|
||||
name = "all";
|
||||
checks = lib.flatten (lib.concatMap
|
||||
(pkg: lib.optional (pkg ? checks) (lib.filter lib.isDerivation (lib.attrValues pkg.checks)))
|
||||
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)));
|
||||
mixLibraries = lib.concatMap
|
||||
(pkg: lib.optional (pkg.components ? library) pkg.components.library)
|
||||
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
|
||||
ghc = project.pkg-set.config.ghc.package;
|
||||
};
|
||||
|
||||
in pkgs.runCommand "project-coverage-report"
|
||||
({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
|
||||
LANG = "en_US.UTF-8";
|
||||
@ -124,30 +137,14 @@ in pkgs.runCommand "project-coverage-report"
|
||||
cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/
|
||||
'') coverageReports)}
|
||||
|
||||
if [ ''${#tixFiles[@]} -ne 0 ]; then
|
||||
# Create tix file with test run information for all packages
|
||||
tixFile="$out/share/hpc/vanilla/tix/all/all.tix"
|
||||
hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile")
|
||||
hpcSumCmd+=("''${tixFiles[@]}")
|
||||
echo "''${hpcSumCmd[@]}"
|
||||
eval "''${hpcSumCmd[@]}"
|
||||
# Copy out "all" coverage report
|
||||
cp -R ${allCoverageReport}/share/hpc/vanilla/tix/all $out/share/hpc/vanilla/tix
|
||||
cp -R ${allCoverageReport}/share/hpc/vanilla/html/all $out/share/hpc/vanilla/html
|
||||
|
||||
# Markup a HTML coverage report for the entire project
|
||||
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
|
||||
echo "report coverage-per-package $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
|
||||
# Markup a HTML coverage summary report for the entire project
|
||||
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
|
||||
|
||||
local markupOutDir="$out/share/hpc/vanilla/html/all"
|
||||
local srcDirs=${toBashArray srcDirs}
|
||||
local mixDirs=${toBashArray mixDirs}
|
||||
local allMixModules=()
|
||||
|
||||
mkdir $markupOutDir
|
||||
findModules allMixModules "$out/share/hpc/vanilla/mix/" "*.mix"
|
||||
|
||||
markup srcDirs mixDirs allMixModules "$markupOutDir" "$tixFile"
|
||||
|
||||
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
|
||||
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
|
||||
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
|
||||
fi
|
||||
echo "report coverage $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
|
||||
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
|
||||
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
|
||||
''
|
||||
|
||||
167
lib/cover.nix
167
lib/cover.nix
@ -1,18 +1,18 @@
|
||||
# The following collects coverage information from a set of given "checks" and
|
||||
# provides a coverage report showing how those "checks" cover a set of given
|
||||
# "mixLibraries".
|
||||
{ stdenv, lib, haskellLib, pkgs }:
|
||||
|
||||
# Name of the coverage report, which should be unique
|
||||
# Name of the coverage report, which should be unique.
|
||||
{ name
|
||||
# Library to check coverage of
|
||||
, library
|
||||
# List of check derivations that generate coverage
|
||||
, checks
|
||||
# List of other libraries to include in the coverage report. The
|
||||
# default value if just the derivation provided as the `library`
|
||||
# argument. Use a larger list of libraries if you would like the tests
|
||||
# of one local package to generate coverage for another.
|
||||
, mixLibraries ? [library]
|
||||
# hack for project-less projects
|
||||
, ghc ? library.project.pkg-set.config.ghc.package
|
||||
# List of check derivations that generate coverage.
|
||||
, checks ? []
|
||||
# List of libraries to include in the coverage report. If one of the above
|
||||
# checks generates coverage for a particular library, coverage will only
|
||||
# be included if that library is in this list.
|
||||
, mixLibraries ? []
|
||||
# Hack for project-less projects.
|
||||
, ghc ? if mixLibraries == [] then null else (lib.head mixLibraries).project.pkg-set.config.ghc.package
|
||||
}:
|
||||
|
||||
let
|
||||
@ -26,7 +26,7 @@ let
|
||||
in pkgs.runCommand (name + "-coverage-report")
|
||||
({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
|
||||
passthru = {
|
||||
inherit name library checks;
|
||||
inherit name checks mixLibraries;
|
||||
};
|
||||
# HPC will fail if the Haskell file contains non-ASCII characters,
|
||||
# unless our locale is set correctly. This has been fixed, but we
|
||||
@ -70,71 +70,105 @@ in pkgs.runCommand (name + "-coverage-report")
|
||||
local -n tixFs=$2
|
||||
local outFile="$3"
|
||||
|
||||
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
|
||||
if (( "''${#tixFs[@]}" > 0 )); then
|
||||
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
|
||||
|
||||
for module in "''${includedModules[@]}"; do
|
||||
hpcSumCmd+=("--include=$module")
|
||||
done
|
||||
for module in "''${includedModules[@]}"; do
|
||||
hpcSumCmd+=("--include=$module")
|
||||
done
|
||||
|
||||
for tixFile in "''${tixFs[@]}"; do
|
||||
hpcSumCmd+=("$tixFile")
|
||||
done
|
||||
for tixFile in "''${tixFs[@]}"; do
|
||||
hpcSumCmd+=("$tixFile")
|
||||
done
|
||||
|
||||
echo "''${hpcSumCmd[@]}"
|
||||
eval "''${hpcSumCmd[@]}"
|
||||
echo "''${hpcSumCmd[@]}"
|
||||
eval "''${hpcSumCmd[@]}"
|
||||
else
|
||||
# If there are no tix files we output an empty tix file so that we can
|
||||
# markup an empty HTML coverage report. This is preferable to failing to
|
||||
# output a HTML report.
|
||||
echo 'Tix []' > $outFile
|
||||
fi
|
||||
}
|
||||
|
||||
function findModules() {
|
||||
local searchDir=$2
|
||||
local -n result=$1
|
||||
local -n searchDirs=$2
|
||||
local pattern=$3
|
||||
|
||||
pushd $searchDir
|
||||
mapfile -d $'\0' $1 < <(find ./ -type f \
|
||||
-wholename "$pattern" -not -name "Paths*" \
|
||||
-exec basename {} \; \
|
||||
| sed "s/\.mix$//" \
|
||||
| tr "\n" "\0")
|
||||
popd
|
||||
for dir in "''${searchDirs[@]}"; do
|
||||
pushd $dir
|
||||
local temp=()
|
||||
mapfile -d $'\0' temp < <(find ./ -type f \
|
||||
-wholename "$pattern" -not -name "Paths*" \
|
||||
-exec basename {} \; \
|
||||
| sed "s/\.mix$//" \
|
||||
| tr "\n" "\0")
|
||||
result+=("''${temp[@]}")
|
||||
popd
|
||||
done
|
||||
}
|
||||
|
||||
local mixDirs=${toBashArray mixDirs}
|
||||
|
||||
mkdir -p $out/nix-support
|
||||
mkdir -p $out/share/hpc/vanilla/mix/${name}
|
||||
mkdir -p $out/share/hpc/vanilla/mix/
|
||||
mkdir -p $out/share/hpc/vanilla/tix/${name}
|
||||
mkdir -p $out/share/hpc/vanilla/html/${name}
|
||||
|
||||
# Copy over mix files verbatim
|
||||
local srcDirs=${toBashArray srcDirs}
|
||||
local mixDirs=${toBashArray mixDirs}
|
||||
|
||||
# Copy out mix files used for this report
|
||||
for dir in "''${mixDirs[@]}"; do
|
||||
if [ -d "$dir" ]; then
|
||||
cp -R "$dir"/* $out/share/hpc/vanilla/mix/${name}
|
||||
cp -R "$dir" $out/share/hpc/vanilla/mix/
|
||||
fi
|
||||
done
|
||||
|
||||
local srcDirs=${toBashArray srcDirs}
|
||||
local allMixModules=()
|
||||
local pkgMixModules=()
|
||||
local mixModules=()
|
||||
# Mix modules for all packages in "mixLibraries"
|
||||
findModules mixModules mixDirs "*.mix"
|
||||
|
||||
# The behaviour of stack coverage reports is to provide tix files
|
||||
# that include coverage information for every local package, but
|
||||
# to provide HTML reports that only include coverage info for the
|
||||
# current package. We emulate the same behaviour here. If the user
|
||||
# includes all local packages in the mix libraries argument, they
|
||||
# will get a coverage report very similar to stack.
|
||||
# We need to make a distinction between library "exposed-modules" and
|
||||
# "other-modules" used in test suites:
|
||||
# - "exposed-modules" are addressed as "$library-$version-$hash/module"
|
||||
# - "other-modules" are addressed as "module"
|
||||
#
|
||||
# This complicates the code required to find the mix modules. For a given mix directory:
|
||||
#
|
||||
# mix
|
||||
# └── ntp-client-0.0.1
|
||||
# └── ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5
|
||||
# ├── Network.NTP.Client.mix
|
||||
# ├── Network.NTP.Client.Packet.mix
|
||||
# └── Network.NTP.Client.Query.mix
|
||||
#
|
||||
# Iff ntp-client uses "other-modules" in a test suite, both:
|
||||
# - "mix/ntp-client-0.0.1", and
|
||||
# - "mix/ntp-client-0.0.1/ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5"
|
||||
# need to be provided to hpc as search directories.
|
||||
#
|
||||
# I'd prefer to just exclude "other-modules", but I can't think of an easy
|
||||
# way to do that in bash.
|
||||
#
|
||||
# Here we expand the search dirs and modify the mix dirs accordingly:
|
||||
for dir in "''${mixDirs[@]}"; do
|
||||
local otherModulesSearchDirs=()
|
||||
# Simply consider any directory with a mix file as a search directory.
|
||||
mapfile -d $'\0' otherModulesSearchDirs < <(find $dir -type f \
|
||||
-wholename "*.mix" \
|
||||
-exec dirname {} \; \
|
||||
| uniq \
|
||||
| tr "\n" "\0")
|
||||
mixDirs+=("''${otherModulesSearchDirs[@]}")
|
||||
done
|
||||
|
||||
# All mix modules
|
||||
findModules allMixModules "$out/share/hpc/vanilla/mix/${name}" "*.mix"
|
||||
# Only mix modules corresponding to this package
|
||||
findModules pkgMixModules "$out/share/hpc/vanilla/mix/${name}" "*${name}*/*.mix"
|
||||
|
||||
# For each test
|
||||
local tixFiles=()
|
||||
${lib.concatStringsSep "\n" (builtins.map (check: ''
|
||||
if [ -d "${check}/share/hpc/vanilla/tix" ]; then
|
||||
pushd ${check}/share/hpc/vanilla/tix
|
||||
|
||||
tixFile="$(find . -iwholename "*.tix" -type f -print -quit)"
|
||||
local newTixFile=$out/share/hpc/vanilla/tix/${name}/"$tixFile"
|
||||
local newTixFile=$out/share/hpc/vanilla/tix/${check.name}/"$(basename $tixFile)"
|
||||
|
||||
mkdir -p "$(dirname $newTixFile)"
|
||||
# Copy over the tix file verbatim
|
||||
@ -143,29 +177,28 @@ in pkgs.runCommand (name + "-coverage-report")
|
||||
# Add the tix file to our list
|
||||
tixFiles+=("$newTixFile")
|
||||
|
||||
# Create a coverage report for *just that test*
|
||||
markup srcDirs mixDirs pkgMixModules "$out/share/hpc/vanilla/html/${name}/${check.exeName}/" "$newTixFile"
|
||||
# Create a coverage report for *just that check* affecting any of the
|
||||
# "mixLibraries"
|
||||
markup srcDirs mixDirs mixModules "$out/share/hpc/vanilla/html/${check.name}/" "$newTixFile"
|
||||
|
||||
popd
|
||||
fi
|
||||
'') checks)
|
||||
}
|
||||
|
||||
# Sum tix files to create a tix file with all relevant tix
|
||||
# information and markup a HTML report from this info.
|
||||
if (( "''${#tixFiles[@]}" > 0 )); then
|
||||
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
|
||||
local markupOutDir="$out/share/hpc/vanilla/html/${name}"
|
||||
# Sum tix files to create a tix file with tix information from all tests in
|
||||
# the package and markup a HTML report from this info.
|
||||
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
|
||||
local markupOutDir="$out/share/hpc/vanilla/html/${name}"
|
||||
|
||||
# Sum all of our tix file, including modules from any local package
|
||||
sumTix allMixModules tixFiles "$sumTixFile"
|
||||
# Sum all of our tix files
|
||||
sumTix mixModules tixFiles "$sumTixFile"
|
||||
|
||||
# Markup a HTML report, included modules from only this package
|
||||
markup srcDirs mixDirs pkgMixModules "$markupOutDir" "$sumTixFile"
|
||||
# Markup a HTML report
|
||||
markup srcDirs mixDirs mixModules "$markupOutDir" "$sumTixFile"
|
||||
|
||||
# Provide a HTML zipfile and Hydra links
|
||||
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
|
||||
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
|
||||
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
|
||||
fi
|
||||
# Provide a HTML zipfile and Hydra links
|
||||
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
|
||||
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
|
||||
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
|
||||
''
|
||||
|
||||
@ -591,8 +591,9 @@ final: prev: {
|
||||
|
||||
coverageReport = haskellLib.coverageReport (rec {
|
||||
name = package.identifier.name + "-" + package.identifier.version;
|
||||
library = if components ? library then components.library else null;
|
||||
# Include the checks for a single package.
|
||||
checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks);
|
||||
# Checks from that package may provide coverage information for any library in the project.
|
||||
mixLibraries = final.lib.concatMap
|
||||
(pkg: final.lib.optional (pkg.components ? library) pkg.components.library)
|
||||
(final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
|
||||
|
||||
Loading…
Reference in New Issue
Block a user