From 0dca71e2f3f3d63eeab01136138e0e1ceab5320d Mon Sep 17 00:00:00 2001 From: Samuel Evans-Powell Date: Tue, 19 Jul 2022 15:40:41 +0800 Subject: [PATCH] Improve coverage reports (#1548) * Provide an empty coverage report when package has no coverage - When covering a project with multiple packages, the project coverage report will produce a table of individual coverage reports for each package. The links in this table don't go anywhere when a package is uncovered (they are broken links). - By producing an empty coverage report when a package has no coverage, we fix the broken links, and provide a tad more information about the coverage state of that package. * Improve project coverage report index page - Provide a link to the union/all coverage report. - Format the reports as a list instead of a table of one column. - Provide better explanation of what each report means. * Add warning explaining modules with no coverage - Modules that have no coverage at all are simply not included in the HTML reports generated by HPC. - Add a warning to the project coverage report index page so users are aware of this limitation of HPC. * Simplify interface of coverageReport - Remove the concept of "package boundaries" from the "coverageReport" function. - The "coverageReport" is now a function of: - arbitrary checks generating tix files - arbitrary mix modules - This more closely reflects the usage of hpc, which doesn't care about package boundaries. - Use this new "coverageReport" function to simplify the "projectCoverageReport" implementation. A project coverage report now simply: - copies out constituent coverage reports. - writes out an "all coverage report" using all checks in the project and all mix modules.. - writes out a summary index page. --- lib/cover-project.nix | 91 +++++++++++------------ lib/cover.nix | 167 +++++++++++++++++++++++++----------------- overlays/haskell.nix | 3 +- 3 files changed, 146 insertions(+), 115 deletions(-) diff --git a/lib/cover-project.nix b/lib/cover-project.nix index 287a6109..1e34c560 100644 --- a/lib/cover-project.nix +++ b/lib/cover-project.nix @@ -10,19 +10,12 @@ coverageReports: let toBashArray = arr: "(" + (lib.concatStringsSep " " arr) + ")"; - # Create table rows for a project coverage index page that look something like: - # - # | Package | - # |------------------| - # | cardano-shell | - # | cardano-launcher | - coverageTableRows = coverageReport: + # Create a list element for a project coverage index page. + coverageListElement = coverageReport: '' - - - ${coverageReport.passthru.name} - - +
  • + ${coverageReport.passthru.name} +
    • ''; projectIndexHtml = pkgs.writeText "index.html" '' @@ -31,23 +24,32 @@ let - - - - - - - ${with lib; concatStringsSep "\n" (map coverageTableRows coverageReports)} - - -
    Report
    +
    + WARNING: Modules with no coverage are not included in any of these reports, this is just how HPC works under the hood. +
    +
    +

    Union Report

    +

    The following report shows how each module is covered by any test in the project:

    + +
    +
    +

    Individual Reports

    +

    The following reports show how the tests of each package cover modules in the project:

    + +
    ''; ghc = project.pkg-set.config.ghc.package; - libs = lib.remove null (map (r: r.library) coverageReports); + libs = lib.unique (lib.concatMap (r: r.mixLibraries) coverageReports); mixDirs = map @@ -56,6 +58,17 @@ let srcDirs = map (l: l.srcSubDirPath) libs; + allCoverageReport = haskellLib.coverageReport { + name = "all"; + checks = lib.flatten (lib.concatMap + (pkg: lib.optional (pkg ? checks) (lib.filter lib.isDerivation (lib.attrValues pkg.checks))) + (lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs))); + mixLibraries = lib.concatMap + (pkg: lib.optional (pkg.components ? library) pkg.components.library) + (lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs)); + ghc = project.pkg-set.config.ghc.package; + }; + in pkgs.runCommand "project-coverage-report" ({ nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ]; LANG = "en_US.UTF-8"; @@ -124,30 +137,14 @@ in pkgs.runCommand "project-coverage-report" cp -R $report/share/hpc/vanilla/html/* $out/share/hpc/vanilla/html/ '') coverageReports)} - if [ ''${#tixFiles[@]} -ne 0 ]; then - # Create tix file with test run information for all packages - tixFile="$out/share/hpc/vanilla/tix/all/all.tix" - hpcSumCmd=("hpc" "sum" "--union" "--output=$tixFile") - hpcSumCmd+=("''${tixFiles[@]}") - echo "''${hpcSumCmd[@]}" - eval "''${hpcSumCmd[@]}" + # Copy out "all" coverage report + cp -R ${allCoverageReport}/share/hpc/vanilla/tix/all $out/share/hpc/vanilla/tix + cp -R ${allCoverageReport}/share/hpc/vanilla/html/all $out/share/hpc/vanilla/html - # Markup a HTML coverage report for the entire project - cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html - echo "report coverage-per-package $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products + # Markup a HTML coverage summary report for the entire project + cp ${projectIndexHtml} $out/share/hpc/vanilla/html/index.html - local markupOutDir="$out/share/hpc/vanilla/html/all" - local srcDirs=${toBashArray srcDirs} - local mixDirs=${toBashArray mixDirs} - local allMixModules=() - - mkdir $markupOutDir - findModules allMixModules "$out/share/hpc/vanilla/mix/" "*.mix" - - markup srcDirs mixDirs allMixModules "$markupOutDir" "$tixFile" - - echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products - ( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . ) - echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products - fi + echo "report coverage $out/share/hpc/vanilla/html/index.html" >> $out/nix-support/hydra-build-products + ( cd $out/share/hpc/vanilla/html ; zip -r $out/share/hpc/vanilla/html.zip . ) + echo "file zip $out/share/hpc/vanilla/html.zip" >> $out/nix-support/hydra-build-products '' diff --git a/lib/cover.nix b/lib/cover.nix index 4d17eae0..90fd8bf7 100644 --- a/lib/cover.nix +++ b/lib/cover.nix @@ -1,18 +1,18 @@ +# The following collects coverage information from a set of given "checks" and +# provides a coverage report showing how those "checks" cover a set of given +# "mixLibraries". { stdenv, lib, haskellLib, pkgs }: -# Name of the coverage report, which should be unique +# Name of the coverage report, which should be unique. { name -# Library to check coverage of -, library -# List of check derivations that generate coverage -, checks -# List of other libraries to include in the coverage report. The -# default value if just the derivation provided as the `library` -# argument. Use a larger list of libraries if you would like the tests -# of one local package to generate coverage for another. -, mixLibraries ? [library] -# hack for project-less projects -, ghc ? library.project.pkg-set.config.ghc.package +# List of check derivations that generate coverage. +, checks ? [] +# List of libraries to include in the coverage report. If one of the above +# checks generates coverage for a particular library, coverage will only +# be included if that library is in this list. +, mixLibraries ? [] +# Hack for project-less projects. +, ghc ? if mixLibraries == [] then null else (lib.head mixLibraries).project.pkg-set.config.ghc.package }: let @@ -26,7 +26,7 @@ let in pkgs.runCommand (name + "-coverage-report") ({nativeBuildInputs = [ (ghc.buildGHC or ghc) pkgs.buildPackages.zip ]; passthru = { - inherit name library checks; + inherit name checks mixLibraries; }; # HPC will fail if the Haskell file contains non-ASCII characters, # unless our locale is set correctly. This has been fixed, but we @@ -70,71 +70,105 @@ in pkgs.runCommand (name + "-coverage-report") local -n tixFs=$2 local outFile="$3" - local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile") + if (( "''${#tixFs[@]}" > 0 )); then + local hpcSumCmd=("hpc" "sum" "--union" "--output=$outFile") - for module in "''${includedModules[@]}"; do - hpcSumCmd+=("--include=$module") - done + for module in "''${includedModules[@]}"; do + hpcSumCmd+=("--include=$module") + done - for tixFile in "''${tixFs[@]}"; do - hpcSumCmd+=("$tixFile") - done + for tixFile in "''${tixFs[@]}"; do + hpcSumCmd+=("$tixFile") + done - echo "''${hpcSumCmd[@]}" - eval "''${hpcSumCmd[@]}" + echo "''${hpcSumCmd[@]}" + eval "''${hpcSumCmd[@]}" + else + # If there are no tix files we output an empty tix file so that we can + # markup an empty HTML coverage report. This is preferable to failing to + # output a HTML report. + echo 'Tix []' > $outFile + fi } function findModules() { - local searchDir=$2 + local -n result=$1 + local -n searchDirs=$2 local pattern=$3 - pushd $searchDir - mapfile -d $'\0' $1 < <(find ./ -type f \ - -wholename "$pattern" -not -name "Paths*" \ - -exec basename {} \; \ - | sed "s/\.mix$//" \ - | tr "\n" "\0") - popd + for dir in "''${searchDirs[@]}"; do + pushd $dir + local temp=() + mapfile -d $'\0' temp < <(find ./ -type f \ + -wholename "$pattern" -not -name "Paths*" \ + -exec basename {} \; \ + | sed "s/\.mix$//" \ + | tr "\n" "\0") + result+=("''${temp[@]}") + popd + done } - local mixDirs=${toBashArray mixDirs} - mkdir -p $out/nix-support - mkdir -p $out/share/hpc/vanilla/mix/${name} + mkdir -p $out/share/hpc/vanilla/mix/ mkdir -p $out/share/hpc/vanilla/tix/${name} mkdir -p $out/share/hpc/vanilla/html/${name} - # Copy over mix files verbatim + local srcDirs=${toBashArray srcDirs} + local mixDirs=${toBashArray mixDirs} + + # Copy out mix files used for this report for dir in "''${mixDirs[@]}"; do if [ -d "$dir" ]; then - cp -R "$dir"/* $out/share/hpc/vanilla/mix/${name} + cp -R "$dir" $out/share/hpc/vanilla/mix/ fi done - local srcDirs=${toBashArray srcDirs} - local allMixModules=() - local pkgMixModules=() + local mixModules=() + # Mix modules for all packages in "mixLibraries" + findModules mixModules mixDirs "*.mix" - # The behaviour of stack coverage reports is to provide tix files - # that include coverage information for every local package, but - # to provide HTML reports that only include coverage info for the - # current package. We emulate the same behaviour here. If the user - # includes all local packages in the mix libraries argument, they - # will get a coverage report very similar to stack. + # We need to make a distinction between library "exposed-modules" and + # "other-modules" used in test suites: + # - "exposed-modules" are addressed as "$library-$version-$hash/module" + # - "other-modules" are addressed as "module" + # + # This complicates the code required to find the mix modules. For a given mix directory: + # + # mix + # └── ntp-client-0.0.1 + # └── ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5 + # ├── Network.NTP.Client.mix + # ├── Network.NTP.Client.Packet.mix + # └── Network.NTP.Client.Query.mix + # + # Iff ntp-client uses "other-modules" in a test suite, both: + # - "mix/ntp-client-0.0.1", and + # - "mix/ntp-client-0.0.1/ntp-client-0.0.1-gYjRsBHUCaHX7ENcjHnw5" + # need to be provided to hpc as search directories. + # + # I'd prefer to just exclude "other-modules", but I can't think of an easy + # way to do that in bash. + # + # Here we expand the search dirs and modify the mix dirs accordingly: + for dir in "''${mixDirs[@]}"; do + local otherModulesSearchDirs=() + # Simply consider any directory with a mix file as a search directory. + mapfile -d $'\0' otherModulesSearchDirs < <(find $dir -type f \ + -wholename "*.mix" \ + -exec dirname {} \; \ + | uniq \ + | tr "\n" "\0") + mixDirs+=("''${otherModulesSearchDirs[@]}") + done - # All mix modules - findModules allMixModules "$out/share/hpc/vanilla/mix/${name}" "*.mix" - # Only mix modules corresponding to this package - findModules pkgMixModules "$out/share/hpc/vanilla/mix/${name}" "*${name}*/*.mix" - - # For each test local tixFiles=() ${lib.concatStringsSep "\n" (builtins.map (check: '' if [ -d "${check}/share/hpc/vanilla/tix" ]; then pushd ${check}/share/hpc/vanilla/tix tixFile="$(find . -iwholename "*.tix" -type f -print -quit)" - local newTixFile=$out/share/hpc/vanilla/tix/${name}/"$tixFile" + local newTixFile=$out/share/hpc/vanilla/tix/${check.name}/"$(basename $tixFile)" mkdir -p "$(dirname $newTixFile)" # Copy over the tix file verbatim @@ -143,29 +177,28 @@ in pkgs.runCommand (name + "-coverage-report") # Add the tix file to our list tixFiles+=("$newTixFile") - # Create a coverage report for *just that test* - markup srcDirs mixDirs pkgMixModules "$out/share/hpc/vanilla/html/${name}/${check.exeName}/" "$newTixFile" + # Create a coverage report for *just that check* affecting any of the + # "mixLibraries" + markup srcDirs mixDirs mixModules "$out/share/hpc/vanilla/html/${check.name}/" "$newTixFile" popd fi '') checks) } - # Sum tix files to create a tix file with all relevant tix - # information and markup a HTML report from this info. - if (( "''${#tixFiles[@]}" > 0 )); then - local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix" - local markupOutDir="$out/share/hpc/vanilla/html/${name}" + # Sum tix files to create a tix file with tix information from all tests in + # the package and markup a HTML report from this info. + local sumTixFile="$out/share/hpc/vanilla/tix/${name}/${name}.tix" + local markupOutDir="$out/share/hpc/vanilla/html/${name}" - # Sum all of our tix file, including modules from any local package - sumTix allMixModules tixFiles "$sumTixFile" + # Sum all of our tix files + sumTix mixModules tixFiles "$sumTixFile" - # Markup a HTML report, included modules from only this package - markup srcDirs mixDirs pkgMixModules "$markupOutDir" "$sumTixFile" + # Markup a HTML report + markup srcDirs mixDirs mixModules "$markupOutDir" "$sumTixFile" - # Provide a HTML zipfile and Hydra links - ( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . ) - echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products - echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products - fi + # Provide a HTML zipfile and Hydra links + ( cd "$markupOutDir" ; zip -r $out/share/hpc/vanilla/${name}-html.zip . ) + echo "report coverage $markupOutDir/hpc_index.html" >> $out/nix-support/hydra-build-products + echo "file zip $out/share/hpc/vanilla/${name}-html.zip" >> $out/nix-support/hydra-build-products '' diff --git a/overlays/haskell.nix b/overlays/haskell.nix index 426d3927..2efd5c9e 100644 --- a/overlays/haskell.nix +++ b/overlays/haskell.nix @@ -591,8 +591,9 @@ final: prev: { coverageReport = haskellLib.coverageReport (rec { name = package.identifier.name + "-" + package.identifier.version; - library = if components ? library then components.library else null; + # Include the checks for a single package. checks = final.lib.filter (final.lib.isDerivation) (final.lib.attrValues package'.checks); + # Checks from that package may provide coverage information for any library in the project. mixLibraries = final.lib.concatMap (pkg: final.lib.optional (pkg.components ? library) pkg.components.library) (final.lib.attrValues (haskellLib.selectProjectPackages project.hsPkgs));