mirror of
https://github.com/ilyakooo0/haskell.nix.git
synced 2024-09-11 06:55:40 +03:00
Improve coverage reports (#1548)
* Provide an empty coverage report when package has no coverage - When covering a project with multiple packages, the project coverage report will produce a table of individual coverage reports for each package. The links in this table don't go anywhere when a package is uncovered (they are broken links). - By producing an empty coverage report when a package has no coverage, we fix the broken links, and provide a tad more information about the coverage state of that package. * Improve project coverage report index page - Provide a link to the union/all coverage report. - Format the reports as a list instead of a table of one column. - Provide better explanation of what each report means. * Add warning explaining modules with no coverage - Modules that have no coverage at all are simply not included in the HTML reports generated by HPC. - Add a warning to the project coverage report index page so users are aware of this limitation of HPC. * Simplify interface of coverageReport - Remove the concept of "package boundaries" from the "coverageReport" function. - The "coverageReport" is now a function of: - arbitrary checks generating tix files - arbitrary mix modules - This more closely reflects the usage of hpc, which doesn't care about package boundaries. - Use this new "coverageReport" function to simplify the "projectCoverageReport" implementation. A project coverage report now simply: - copies out constituent coverage reports. - writes out an "all coverage report" using all checks in the project and all mix modules.. - writes out a summary index page.
This commit is contained in:
parent
80082aebba
commit
0dca71e2f3
@ -10,19 +10,12 @@ coverageReports:
|
|||||||
let
|
let
|
||||||
toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";
|
toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";
|
||||||
|
|
||||||
# Create table rows for a project coverage index page that look something like:
|
# Create a list element for a project coverage index page.
|
||||||
#
|
coverageListElement = coverageReport:
|
||||||
# | Package |
|
|
||||||
# |------------------|
|
|
||||||
# | cardano-shell |
|
|
||||||
# | cardano-launcher |
|
|
||||||
coverageTableRows = coverageReport:
|
|
||||||
''
|
''
|
||||||
<tr>
|
<li>
|
||||||
<td>
|
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</a>
|
||||||
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</href>
|
</li>
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
'';
|
'';
|
||||||
|
|
||||||
projectIndexHtml = pkgs.writeText "index.html" ''
|
projectIndexHtml = pkgs.writeText "index.html" ''
|
||||||
@ -31,23 +24,32 @@ let
|
|||||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<table border="1" width="100%">
|
<div>
|
||||||
<tbody>
|
WARNING: Modules with no coverage are not included in any of these reports, this is just how HPC works under the hood.
|
||||||
<tr>
|
</div>
|
||||||
<th>Report</th>
|
<div>
|
||||||
</tr>
|
<h2>Union Report</h2>
|
||||||
|
<p>The following report shows how each module is covered by any test in the project:</p>
|
||||||
${with lib; concatStringsSep "\n" (map coverageTableRows coverageReports)}
|
<ul>
|
||||||
|
<li>
|
||||||
</tbody>
|
<a href="all/hpc_index.html">all</a>
|
||||||
</table>
|
</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<h2>Individual Reports</h2>
|
||||||
|
<p>The following reports show how the tests of each package cover modules in the project:</p>
|
||||||
|
<ul>
|
||||||
|
${with lib; concatStringsSep "\n" (map coverageListElement coverageReports)}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
'';
|
'';
|
||||||
|
|
||||||
ghc = project.pkg-set.config.ghc.package;
|
ghc = project.pkg-set.config.ghc.package;
|
||||||
|
|
||||||
libs = lib.remove null (map (r: r.library) coverageReports);
|
libs = lib.unique (lib.concatMap (r: r.mixLibraries) coverageReports);
|
||||||
|
|
||||||
mixDirs =
|
mixDirs =
|
||||||
map
|
map
|
||||||
@ -56,6 +58,17 @@ let
|
|||||||
|
|
||||||
srcDirs = map (l: l.srcSubDirPath) libs;
|
srcDirs = map (l: l.srcSubDirPath) libs;
|
||||||
|
|
||||||
|
allCoverageReport = haskellLib.coverageReport {
|
||||||
|
name = "all";
|
||||||
|
checks = lib.flatten (lib.concatMap
|
||||||
|
(pkg: lib.optional (pkg ? checks) (lib.filter lib.isDerivation (lib.attrValues pkg.checks)))
|
||||||
|
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)));
|
||||||
|
mixLibraries = lib.concatMap
|
||||||
|
(pkg: lib.optional (pkg.components ? library) pkg.components.library)
|
||||||
|
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
|
||||||
|
ghc = project.pkg-set.config.ghc.package;
|
||||||
|
};
|
||||||
|
|
||||||
in pkgs.runCommand "project-coverage-report"
|
in pkgs.runCommand "project-coverage-report"
|
||||||
({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
|
({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
|
||||||
LANG = "en_US.UTF-8";
|
LANG = "en_US.UTF-8";
|
||||||
@ -124,30 +137,14 @@ in pkgs.runCommand "project-coverage-report"
|
|||||||
cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/
|
cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/
|
||||||
'') coverageReports)}
|
'') coverageReports)}
|
||||||
|
|
||||||
if [ ''${#tixFiles[@]} -ne 0 ]; then
|
# Copy out "all" coverage report
|
||||||
# Create tix file with test run information for all packages
|
cp -R ${allCoverageReport}/share/hpc/vanilla/tix/all $out/share/hpc/vanilla/tix
|
||||||
tixFile="$out/share/hpc/vanilla/tix/all/all.tix"
|
cp -R ${allCoverageReport}/share/hpc/vanilla/html/all $out/share/hpc/vanilla/html
|
||||||
hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile")
|
|
||||||
hpcSumCmd+=("''${tixFiles[@]}")
|
|
||||||
echo "''${hpcSumCmd[@]}"
|
|
||||||
eval "''${hpcSumCmd[@]}"
|
|
||||||
|
|
||||||
# Markup a HTML coverage report for the entire project
|
# Markup a HTML coverage summary report for the entire project
|
||||||
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
|
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
|
||||||
echo "report coverage-per-package $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
|
|
||||||
|
|
||||||
local markupOutDir="$out/share/hpc/vanilla/html/all"
|
echo "report coverage $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
|
||||||
local srcDirs=${toBashArray srcDirs}
|
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
|
||||||
local mixDirs=${toBashArray mixDirs}
|
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
|
||||||
local allMixModules=()
|
|
||||||
|
|
||||||
mkdir $markupOutDir
|
|
||||||
findModules allMixModules "$out/share/hpc/vanilla/mix/" "*.mix"
|
|
||||||
|
|
||||||
markup srcDirs mixDirs allMixModules "$markupOutDir" "$tixFile"
|
|
||||||
|
|
||||||
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
|
|
||||||
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
|
|
||||||
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
|
|
||||||
fi
|
|
||||||
''
|
''
|
||||||
|
167
lib/cover.nix
167
lib/cover.nix
@ -1,18 +1,18 @@
|
|||||||
|
# The following collects coverage information from a set of given "checks" and
|
||||||
|
# provides a coverage report showing how those "checks" cover a set of given
|
||||||
|
# "mixLibraries".
|
||||||
{ stdenv, lib, haskellLib, pkgs }:
|
{ stdenv, lib, haskellLib, pkgs }:
|
||||||
|
|
||||||
# Name of the coverage report, which should be unique
|
# Name of the coverage report, which should be unique.
|
||||||
{ name
|
{ name
|
||||||
# Library to check coverage of
|
# List of check derivations that generate coverage.
|
||||||
, library
|
, checks ? []
|
||||||
# List of check derivations that generate coverage
|
# List of libraries to include in the coverage report. If one of the above
|
||||||
, checks
|
# checks generates coverage for a particular library, coverage will only
|
||||||
# List of other libraries to include in the coverage report. The
|
# be included if that library is in this list.
|
||||||
# default value if just the derivation provided as the `library`
|
, mixLibraries ? []
|
||||||
# argument. Use a larger list of libraries if you would like the tests
|
# Hack for project-less projects.
|
||||||
# of one local package to generate coverage for another.
|
, ghc ? if mixLibraries == [] then null else (lib.head mixLibraries).project.pkg-set.config.ghc.package
|
||||||
, mixLibraries ? [library]
|
|
||||||
# hack for project-less projects
|
|
||||||
, ghc ? library.project.pkg-set.config.ghc.package
|
|
||||||
}:
|
}:
|
||||||
|
|
||||||
let
|
let
|
||||||
@ -26,7 +26,7 @@ let
|
|||||||
in pkgs.runCommand (name + "-coverage-report")
|
in pkgs.runCommand (name + "-coverage-report")
|
||||||
({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
|
({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
|
||||||
passthru = {
|
passthru = {
|
||||||
inherit name library checks;
|
inherit name checks mixLibraries;
|
||||||
};
|
};
|
||||||
# HPC will fail if the Haskell file contains non-ASCII characters,
|
# HPC will fail if the Haskell file contains non-ASCII characters,
|
||||||
# unless our locale is set correctly. This has been fixed, but we
|
# unless our locale is set correctly. This has been fixed, but we
|
||||||
@ -70,71 +70,105 @@ in pkgs.runCommand (name + "-coverage-report")
|
|||||||
local -n tixFs=$2
|
local -n tixFs=$2
|
||||||
local outFile="$3"
|
local outFile="$3"
|
||||||
|
|
||||||
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
|
if (( "''${#tixFs[@]}" > 0 )); then
|
||||||
|
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
|
||||||
|
|
||||||
for module in "''${includedModules[@]}"; do
|
for module in "''${includedModules[@]}"; do
|
||||||
hpcSumCmd+=("--include=$module")
|
hpcSumCmd+=("--include=$module")
|
||||||
done
|
done
|
||||||
|
|
||||||
for tixFile in "''${tixFs[@]}"; do
|
for tixFile in "''${tixFs[@]}"; do
|
||||||
hpcSumCmd+=("$tixFile")
|
hpcSumCmd+=("$tixFile")
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "''${hpcSumCmd[@]}"
|
echo "''${hpcSumCmd[@]}"
|
||||||
eval "''${hpcSumCmd[@]}"
|
eval "''${hpcSumCmd[@]}"
|
||||||
|
else
|
||||||
|
# If there are no tix files we output an empty tix file so that we can
|
||||||
|
# markup an empty HTML coverage report. This is preferable to failing to
|
||||||
|
# output a HTML report.
|
||||||
|
echo 'Tix []' > $outFile
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function findModules() {
|
function findModules() {
|
||||||
local searchDir=$2
|
local -n result=$1
|
||||||
|
local -n searchDirs=$2
|
||||||
local pattern=$3
|
local pattern=$3
|
||||||
|
|
||||||
pushd $searchDir
|
for dir in "''${searchDirs[@]}"; do
|
||||||
mapfile -d $'\0' $1 < <(find ./ -type f \
|
pushd $dir
|
||||||
-wholename "$pattern" -not -name "Paths*" \
|
local temp=()
|
||||||
-exec basename {} \; \
|
mapfile -d $'\0' temp < <(find ./ -type f \
|
||||||
| sed "s/\.mix$//" \
|
-wholename "$pattern" -not -name "Paths*" \
|
||||||
| tr "\n" "\0")
|
-exec basename {} \; \
|
||||||
popd
|
| sed "s/\.mix$//" \
|
||||||
|
| tr "\n" "\0")
|
||||||
|
result+=("''${temp[@]}")
|
||||||
|
popd
|
||||||
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
local mixDirs=${toBashArray mixDirs}
|
|
||||||
|
|
||||||
mkdir -p $out/nix-support
|
mkdir -p $out/nix-support
|
||||||
mkdir -p $out/share/hpc/vanilla/mix/${name}
|
mkdir -p $out/share/hpc/vanilla/mix/
|
||||||
mkdir -p $out/share/hpc/vanilla/tix/${name}
|
mkdir -p $out/share/hpc/vanilla/tix/${name}
|
||||||
mkdir -p $out/share/hpc/vanilla/html/${name}
|
mkdir -p $out/share/hpc/vanilla/html/${name}
|
||||||
|
|
||||||
# Copy over mix files verbatim
|
local srcDirs=${toBashArray srcDirs}
|
||||||
|
local mixDirs=${toBashArray mixDirs}
|
||||||
|
|
||||||
|
# Copy out mix files used for this report
|
||||||
for dir in "''${mixDirs[@]}"; do
|
for dir in "''${mixDirs[@]}"; do
|
||||||
if [ -d "$dir" ]; then
|
if [ -d "$dir" ]; then
|
||||||
cp -R "$dir"/* $out/share/hpc/vanilla/mix/${name}
|
cp -R "$dir" $out/share/hpc/vanilla/mix/
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
local srcDirs=${toBashArray srcDirs}
|
local mixModules=()
|
||||||
local allMixModules=()
|
# Mix modules for all packages in "mixLibraries"
|
||||||
local pkgMixModules=()
|
findModules mixModules mixDirs "*.mix"
|
||||||
|
|
||||||
# The behaviour of stack coverage reports is to provide tix files
|
# We need to make a distinction between library "exposed-modules" and
|
||||||
# that include coverage information for every local package, but
|
# "other-modules" used in test suites:
|
||||||
# to provide HTML reports that only include coverage info for the
|
# - "exposed-modules" are addressed as "$library-$version-$hash/module"
|
||||||
# current package. We emulate the same behaviour here. If the user
|
# - "other-modules" are addressed as "module"
|
||||||
# includes all local packages in the mix libraries argument, they
|
#
|
||||||
# will get a coverage report very similar to stack.
|
# This complicates the code required to find the mix modules. For a given mix directory:
|
||||||
|
#
|
||||||
|
# mix
|
||||||
|
# └── ntp-client-0.0.1
|
||||||
|
# └── ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5
|
||||||
|
# ├── Network.NTP.Client.mix
|
||||||
|
# ├── Network.NTP.Client.Packet.mix
|
||||||
|
# └── Network.NTP.Client.Query.mix
|
||||||
|
#
|
||||||
|
# Iff ntp-client uses "other-modules" in a test suite, both:
|
||||||
|
# - "mix/ntp-client-0.0.1", and
|
||||||
|
# - "mix/ntp-client-0.0.1/ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5"
|
||||||
|
# need to be provided to hpc as search directories.
|
||||||
|
#
|
||||||
|
# I'd prefer to just exclude "other-modules", but I can't think of an easy
|
||||||
|
# way to do that in bash.
|
||||||
|
#
|
||||||
|
# Here we expand the search dirs and modify the mix dirs accordingly:
|
||||||
|
for dir in "''${mixDirs[@]}"; do
|
||||||
|
local otherModulesSearchDirs=()
|
||||||
|
# Simply consider any directory with a mix file as a search directory.
|
||||||
|
mapfile -d $'\0' otherModulesSearchDirs < <(find $dir -type f \
|
||||||
|
-wholename "*.mix" \
|
||||||
|
-exec dirname {} \; \
|
||||||
|
| uniq \
|
||||||
|
| tr "\n" "\0")
|
||||||
|
mixDirs+=("''${otherModulesSearchDirs[@]}")
|
||||||
|
done
|
||||||
|
|
||||||
# All mix modules
|
|
||||||
findModules allMixModules "$out/share/hpc/vanilla/mix/${name}" "*.mix"
|
|
||||||
# Only mix modules corresponding to this package
|
|
||||||
findModules pkgMixModules "$out/share/hpc/vanilla/mix/${name}" "*${name}*/*.mix"
|
|
||||||
|
|
||||||
# For each test
|
|
||||||
local tixFiles=()
|
local tixFiles=()
|
||||||
${lib.concatStringsSep "\n" (builtins.map (check: ''
|
${lib.concatStringsSep "\n" (builtins.map (check: ''
|
||||||
if [ -d "${check}/share/hpc/vanilla/tix" ]; then
|
if [ -d "${check}/share/hpc/vanilla/tix" ]; then
|
||||||
pushd ${check}/share/hpc/vanilla/tix
|
pushd ${check}/share/hpc/vanilla/tix
|
||||||
|
|
||||||
tixFile="$(find . -iwholename "*.tix" -type f -print -quit)"
|
tixFile="$(find . -iwholename "*.tix" -type f -print -quit)"
|
||||||
local newTixFile=$out/share/hpc/vanilla/tix/${name}/"$tixFile"
|
local newTixFile=$out/share/hpc/vanilla/tix/${check.name}/"$(basename $tixFile)"
|
||||||
|
|
||||||
mkdir -p "$(dirname $newTixFile)"
|
mkdir -p "$(dirname $newTixFile)"
|
||||||
# Copy over the tix file verbatim
|
# Copy over the tix file verbatim
|
||||||
@ -143,29 +177,28 @@ in pkgs.runCommand (name + "-coverage-report")
|
|||||||
# Add the tix file to our list
|
# Add the tix file to our list
|
||||||
tixFiles+=("$newTixFile")
|
tixFiles+=("$newTixFile")
|
||||||
|
|
||||||
# Create a coverage report for *just that test*
|
# Create a coverage report for *just that check* affecting any of the
|
||||||
markup srcDirs mixDirs pkgMixModules "$out/share/hpc/vanilla/html/${name}/${check.exeName}/" "$newTixFile"
|
# "mixLibraries"
|
||||||
|
markup srcDirs mixDirs mixModules "$out/share/hpc/vanilla/html/${check.name}/" "$newTixFile"
|
||||||
|
|
||||||
popd
|
popd
|
||||||
fi
|
fi
|
||||||
'') checks)
|
'') checks)
|
||||||
}
|
}
|
||||||
|
|
||||||
# Sum tix files to create a tix file with all relevant tix
|
# Sum tix files to create a tix file with tix information from all tests in
|
||||||
# information and markup a HTML report from this info.
|
# the package and markup a HTML report from this info.
|
||||||
if (( "''${#tixFiles[@]}" > 0 )); then
|
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
|
||||||
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
|
local markupOutDir="$out/share/hpc/vanilla/html/${name}"
|
||||||
local markupOutDir="$out/share/hpc/vanilla/html/${name}"
|
|
||||||
|
|
||||||
# Sum all of our tix file, including modules from any local package
|
# Sum all of our tix files
|
||||||
sumTix allMixModules tixFiles "$sumTixFile"
|
sumTix mixModules tixFiles "$sumTixFile"
|
||||||
|
|
||||||
# Markup a HTML report, included modules from only this package
|
# Markup a HTML report
|
||||||
markup srcDirs mixDirs pkgMixModules "$markupOutDir" "$sumTixFile"
|
markup srcDirs mixDirs mixModules "$markupOutDir" "$sumTixFile"
|
||||||
|
|
||||||
# Provide a HTML zipfile and Hydra links
|
# Provide a HTML zipfile and Hydra links
|
||||||
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
|
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
|
||||||
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
|
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
|
||||||
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
|
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
|
||||||
fi
|
|
||||||
''
|
''
|
||||||
|
@ -591,8 +591,9 @@ final: prev: {
|
|||||||
|
|
||||||
coverageReport = haskellLib.coverageReport (rec {
|
coverageReport = haskellLib.coverageReport (rec {
|
||||||
name = package.identifier.name + "-" + package.identifier.version;
|
name = package.identifier.name + "-" + package.identifier.version;
|
||||||
library = if components ? library then components.library else null;
|
# Include the checks for a single package.
|
||||||
checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks);
|
checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks);
|
||||||
|
# Checks from that package may provide coverage information for any library in the project.
|
||||||
mixLibraries = final.lib.concatMap
|
mixLibraries = final.lib.concatMap
|
||||||
(pkg: final.lib.optional (pkg.components ? library) pkg.components.library)
|
(pkg: final.lib.optional (pkg.components ? library) pkg.components.library)
|
||||||
(final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
|
(final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
|
||||||
|
Loading…
Reference in New Issue
Block a user