ilyakooo0/haskell.nix
1
0
Fork 0
mirror of https://github.com/ilyakooo0/haskell.nix.git synced 2024-09-11 06:55:40 +03:00
Code Issues Projects Releases Wiki Activity

Improve coverage reports (#1548)

Browse Source 
* Provide an empty coverage report when package has no coverage

- When covering a project with multiple packages, the project coverage report
  will produce a table of individual coverage reports for each package. The
  links in this table don't go anywhere when a package is uncovered (they are
  broken links).
  - By producing an empty coverage report when a package has no coverage, we fix
    the broken links, and provide a tad more information about the coverage
    state of that package.

* Improve project coverage report index page

- Provide a link to the union/all coverage report.
- Format the reports as a list instead of a table of one column.
- Provide better explanation of what each report means.

* Add warning explaining modules with no coverage

- Modules that have no coverage at all are simply not included in the HTML
  reports generated by HPC.
- Add a warning to the project coverage report index page so users are aware of
  this limitation of HPC.

* Simplify interface of coverageReport

- Remove the concept of "package boundaries" from the "coverageReport" function.
- The "coverageReport" is now a function of:
  - arbitrary checks generating tix files
  - arbitrary mix modules
- This more closely reflects the usage of hpc, which doesn't care about package
  boundaries.
- Use this new "coverageReport" function to simplify the "projectCoverageReport"
  implementation. A project coverage report now simply:
  - copies out constituent coverage reports.
  - writes out an "all coverage report" using all checks in the project and all
    mix modules..
  - writes out a summary index page.
This commit is contained in:
Samuel Evans-Powell 2022-07-19 15:40:41 +08:00 committed by GitHub
parent 80082aebba
commit 0dca71e2f3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 146 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
lib/cover-project.nix
View File

@ -10,19 +10,12 @@ coverageReports:
let let
toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")"; toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")";
# Create table rows for a project coverage index page that look something like: # Create a list element for a project coverage index page.
# coverageListElement = coverageReport:
# | Package |
# |------------------|
# | cardano-shell |
# | cardano-launcher |
coverageTableRows = coverageReport:
'' ''
<tr> <li>
<td> <a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</a>
<a href="${coverageReport.passthru.name}/hpc_index.html">${coverageReport.passthru.name}</href> </li>
</td>
</tr>
''; '';
projectIndexHtml = pkgs.writeText "index.html" '' projectIndexHtml = pkgs.writeText "index.html" ''
@ -31,23 +24,32 @@ let
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
</head> </head>
<body> <body>
<table border="1" width="100%"> <div>
<tbody> WARNING: Modules with no coverage are not included in any of these reports, this is just how HPC works under the hood.
<tr> </div>
<th>Report</th> <div>
</tr> <h2>Union Report</h2>
<p>The following report shows how each module is covered by any test in the project:</p>
${with lib; concatStringsSep "\n" (map coverageTableRows coverageReports)} <ul>
<li>
</tbody> <a href="all/hpc_index.html">all</a>
</table> </li>
</ul>
</div>
<div>
<h2>Individual Reports</h2>
<p>The following reports show how the tests of each package cover modules in the project:</p>
<ul>
${with lib; concatStringsSep "\n" (map coverageListElement coverageReports)}
</ul>
</div>
</body> </body>
</html> </html>
''; '';
ghc = project.pkg-set.config.ghc.package; ghc = project.pkg-set.config.ghc.package;
libs = lib.remove null (map (r: r.library) coverageReports); libs = lib.unique (lib.concatMap (r: r.mixLibraries) coverageReports);
mixDirs = mixDirs =
map map
@ -56,6 +58,17 @@ let
srcDirs = map (l: l.srcSubDirPath) libs; srcDirs = map (l: l.srcSubDirPath) libs;
allCoverageReport = haskellLib.coverageReport {
name = "all";
checks = lib.flatten (lib.concatMap
(pkg: lib.optional (pkg ? checks) (lib.filter lib.isDerivation (lib.attrValues pkg.checks)))
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)));
mixLibraries = lib.concatMap
(pkg: lib.optional (pkg.components ? library) pkg.components.library)
(lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));
ghc = project.pkg-set.config.ghc.package;
};
in pkgs.runCommand "project-coverage-report" in pkgs.runCommand "project-coverage-report"
({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ]; ({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
LANG = "en_US.UTF-8"; LANG = "en_US.UTF-8";
@ -124,30 +137,14 @@ in pkgs.runCommand "project-coverage-report"
cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/ cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/
'') coverageReports)} '') coverageReports)}
if [ ''${#tixFiles[@]} -ne 0 ]; then # Copy out "all" coverage report
# Create tix file with test run information for all packages cp -R ${allCoverageReport}/share/hpc/vanilla/tix/all $out/share/hpc/vanilla/tix
tixFile="$out/share/hpc/vanilla/tix/all/all.tix" cp -R ${allCoverageReport}/share/hpc/vanilla/html/all $out/share/hpc/vanilla/html
hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile")
hpcSumCmd+=("''${tixFiles[@]}")
echo "''${hpcSumCmd[@]}"
eval "''${hpcSumCmd[@]}"
# Markup a HTML coverage report for the entire project # Markup a HTML coverage summary report for the entire project
cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html
echo "report coverage-per-package $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
local markupOutDir="$out/share/hpc/vanilla/html/all" echo "report coverage $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products
local srcDirs=${toBashArray srcDirs} ( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
local mixDirs=${toBashArray mixDirs} echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
local allMixModules=()
mkdir $markupOutDir
findModules allMixModules "$out/share/hpc/vanilla/mix/" "*.mix"
markup srcDirs mixDirs allMixModules "$markupOutDir" "$tixFile"
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . )
echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products
fi
'' ''

167
lib/cover.nix
View File

@ -1,18 +1,18 @@
# The following collects coverage information from a set of given "checks" and
# provides a coverage report showing how those "checks" cover a set of given
# "mixLibraries".
{ stdenv, lib, haskellLib, pkgs }: { stdenv, lib, haskellLib, pkgs }:
# Name of the coverage report, which should be unique # Name of the coverage report, which should be unique.
{ name { name
# Library to check coverage of # List of check derivations that generate coverage.
, library , checks ? []
# List of check derivations that generate coverage # List of libraries to include in the coverage report. If one of the above
, checks # checks generates coverage for a particular library, coverage will only
# List of other libraries to include in the coverage report. The # be included if that library is in this list.
# default value if just the derivation provided as the `library` , mixLibraries ? []
# argument. Use a larger list of libraries if you would like the tests # Hack for project-less projects.
# of one local package to generate coverage for another. , ghc ? if mixLibraries == [] then null else (lib.head mixLibraries).project.pkg-set.config.ghc.package
, mixLibraries ? [library]
# hack for project-less projects
, ghc ? library.project.pkg-set.config.ghc.package
}: }:
let let
@ -26,7 +26,7 @@ let
in pkgs.runCommand (name + "-coverage-report") in pkgs.runCommand (name + "-coverage-report")
({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ]; ({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ];
passthru = { passthru = {
inherit name library checks; inherit name checks mixLibraries;
}; };
# HPC will fail if the Haskell file contains non-ASCII characters, # HPC will fail if the Haskell file contains non-ASCII characters,
# unless our locale is set correctly. This has been fixed, but we # unless our locale is set correctly. This has been fixed, but we
@ -70,71 +70,105 @@ in pkgs.runCommand (name + "-coverage-report")
local -n tixFs=$2 local -n tixFs=$2
local outFile="$3" local outFile="$3"
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile") if (( "''${#tixFs[@]}" > 0 )); then
local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile")
for module in "''${includedModules[@]}"; do for module in "''${includedModules[@]}"; do
hpcSumCmd+=("--include=$module") hpcSumCmd+=("--include=$module")
done done
for tixFile in "''${tixFs[@]}"; do for tixFile in "''${tixFs[@]}"; do
hpcSumCmd+=("$tixFile") hpcSumCmd+=("$tixFile")
done done
echo "''${hpcSumCmd[@]}" echo "''${hpcSumCmd[@]}"
eval "''${hpcSumCmd[@]}" eval "''${hpcSumCmd[@]}"
else
# If there are no tix files we output an empty tix file so that we can
# markup an empty HTML coverage report. This is preferable to failing to
# output a HTML report.
echo 'Tix []' > $outFile
fi
} }
function findModules() { function findModules() {
local searchDir=$2 local -n result=$1
local -n searchDirs=$2
local pattern=$3 local pattern=$3
pushd $searchDir for dir in "''${searchDirs[@]}"; do
mapfile -d $'\0' $1 < <(find ./ -type f \ pushd $dir
-wholename "$pattern" -not -name "Paths*" \ local temp=()
-exec basename {} \; \ mapfile -d $'\0' temp < <(find ./ -type f \
| sed "s/\.mix$//" \ -wholename "$pattern" -not -name "Paths*" \
| tr "\n" "\0") -exec basename {} \; \
popd | sed "s/\.mix$//" \
| tr "\n" "\0")
result+=("''${temp[@]}")
popd
done
} }
local mixDirs=${toBashArray mixDirs}
mkdir -p $out/nix-support mkdir -p $out/nix-support
mkdir -p $out/share/hpc/vanilla/mix/${name} mkdir -p $out/share/hpc/vanilla/mix/
mkdir -p $out/share/hpc/vanilla/tix/${name} mkdir -p $out/share/hpc/vanilla/tix/${name}
mkdir -p $out/share/hpc/vanilla/html/${name} mkdir -p $out/share/hpc/vanilla/html/${name}
# Copy over mix files verbatim local srcDirs=${toBashArray srcDirs}
local mixDirs=${toBashArray mixDirs}
# Copy out mix files used for this report
for dir in "''${mixDirs[@]}"; do for dir in "''${mixDirs[@]}"; do
if [ -d "$dir" ]; then if [ -d "$dir" ]; then
cp -R "$dir"/* $out/share/hpc/vanilla/mix/${name} cp -R "$dir" $out/share/hpc/vanilla/mix/
fi fi
done done
local srcDirs=${toBashArray srcDirs} local mixModules=()
local allMixModules=() # Mix modules for all packages in "mixLibraries"
local pkgMixModules=() findModules mixModules mixDirs "*.mix"
# The behaviour of stack coverage reports is to provide tix files # We need to make a distinction between library "exposed-modules" and
# that include coverage information for every local package, but # "other-modules" used in test suites:
# to provide HTML reports that only include coverage info for the # - "exposed-modules" are addressed as "$library-$version-$hash/module"
# current package. We emulate the same behaviour here. If the user # - "other-modules" are addressed as "module"
# includes all local packages in the mix libraries argument, they #
# will get a coverage report very similar to stack. # This complicates the code required to find the mix modules. For a given mix directory:
#
# mix
# └── ntp-client-0.0.1
# └── ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5
# ├── Network.NTP.Client.mix
# ├── Network.NTP.Client.Packet.mix
# └── Network.NTP.Client.Query.mix
#
# Iff ntp-client uses "other-modules" in a test suite, both:
# - "mix/ntp-client-0.0.1", and
# - "mix/ntp-client-0.0.1/ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5"
# need to be provided to hpc as search directories.
#
# I'd prefer to just exclude "other-modules", but I can't think of an easy
# way to do that in bash.
#
# Here we expand the search dirs and modify the mix dirs accordingly:
for dir in "''${mixDirs[@]}"; do
local otherModulesSearchDirs=()
# Simply consider any directory with a mix file as a search directory.
mapfile -d $'\0' otherModulesSearchDirs < <(find $dir -type f \
-wholename "*.mix" \
-exec dirname {} \; \
| uniq \
| tr "\n" "\0")
mixDirs+=("''${otherModulesSearchDirs[@]}")
done
# All mix modules
findModules allMixModules "$out/share/hpc/vanilla/mix/${name}" "*.mix"
# Only mix modules corresponding to this package
findModules pkgMixModules "$out/share/hpc/vanilla/mix/${name}" "*${name}*/*.mix"
# For each test
local tixFiles=() local tixFiles=()
${lib.concatStringsSep "\n" (builtins.map (check: '' ${lib.concatStringsSep "\n" (builtins.map (check: ''
if [ -d "${check}/share/hpc/vanilla/tix" ]; then if [ -d "${check}/share/hpc/vanilla/tix" ]; then
pushd ${check}/share/hpc/vanilla/tix pushd ${check}/share/hpc/vanilla/tix
tixFile="$(find . -iwholename "*.tix" -type f -print -quit)" tixFile="$(find . -iwholename "*.tix" -type f -print -quit)"
local newTixFile=$out/share/hpc/vanilla/tix/${name}/"$tixFile" local newTixFile=$out/share/hpc/vanilla/tix/${check.name}/"$(basename $tixFile)"
mkdir -p "$(dirname $newTixFile)" mkdir -p "$(dirname $newTixFile)"
# Copy over the tix file verbatim # Copy over the tix file verbatim
@ -143,29 +177,28 @@ in pkgs.runCommand (name + "-coverage-report")
# Add the tix file to our list # Add the tix file to our list
tixFiles+=("$newTixFile") tixFiles+=("$newTixFile")
# Create a coverage report for *just that test* # Create a coverage report for *just that check* affecting any of the
markup srcDirs mixDirs pkgMixModules "$out/share/hpc/vanilla/html/${name}/${check.exeName}/" "$newTixFile" # "mixLibraries"
markup srcDirs mixDirs mixModules "$out/share/hpc/vanilla/html/${check.name}/" "$newTixFile"
popd popd
fi fi
'') checks) '') checks)
} }
# Sum tix files to create a tix file with all relevant tix # Sum tix files to create a tix file with tix information from all tests in
# information and markup a HTML report from this info. # the package and markup a HTML report from this info.
if (( "''${#tixFiles[@]}" > 0 )); then local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix"
local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix" local markupOutDir="$out/share/hpc/vanilla/html/${name}"
local markupOutDir="$out/share/hpc/vanilla/html/${name}"
# Sum all of our tix file, including modules from any local package # Sum all of our tix files
sumTix allMixModules tixFiles "$sumTixFile" sumTix mixModules tixFiles "$sumTixFile"
# Markup a HTML report, included modules from only this package # Markup a HTML report
markup srcDirs mixDirs pkgMixModules "$markupOutDir" "$sumTixFile" markup srcDirs mixDirs mixModules "$markupOutDir" "$sumTixFile"
# Provide a HTML zipfile and Hydra links # Provide a HTML zipfile and Hydra links
( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . ) ( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . )
echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products
echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products
fi
'' ''

3
overlays/haskell.nix
View File

@ -591,8 +591,9 @@ final: prev: {
coverageReport = haskellLib.coverageReport (rec { coverageReport = haskellLib.coverageReport (rec {
name = package.identifier.name + "-" + package.identifier.version; name = package.identifier.name + "-" + package.identifier.version;
library = if components ? library then components.library else null; # Include the checks for a single package.
checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks); checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks);
# Checks from that package may provide coverage information for any library in the project.
mixLibraries = final.lib.concatMap mixLibraries = final.lib.concatMap
(pkg: final.lib.optional (pkg.components ? library) pkg.components.library) (pkg: final.lib.optional (pkg.components ? library) pkg.components.library)
(final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)); (final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));