From ce0187b2a4a098d710fa711cfb253e2743e9fed1 Mon Sep 17 00:00:00 2001 From: Samuel Evans-Powell Date: Fri, 11 Dec 2020 16:57:35 +0800 Subject: [PATCH] Add golden coverage test (#964) * Add golden coverage test - Other coverage tests check that the coverage tests can be generated, but don't check the coverage output itself. This commit adds a test to ensure that the coverage output is correct. * Use niv and haskell.nix for hpc-coveralls * Get `sources` from `haskell-nix.callPackages` * Use `builtin: false` for hydra in sources.json * Fix for cross compilation. Co-authored-by: Hamish Mackenzie --- nix/sources.json | 26 + test/coverage-golden/default.nix | 50 + test/coverage-golden/golden.json | 2069 ++++++++++++++++++++++++++++++ test/default.nix | 1 + 4 files changed, 2146 insertions(+) create mode 100644 test/coverage-golden/default.nix create mode 100644 test/coverage-golden/golden.json diff --git a/nix/sources.json b/nix/sources.json index ced81617..10a8a9ee 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -12,6 +12,19 @@ "url": "https://github.com/haskell/cabal/archive/94aaa8e4720081f9c75497e2735b90f6a819b08e.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "cardano-shell": { + "branch": "master", + "builtin": false, + "description": "Node shell, a thin layer for running the node and it's modules.", + "homepage": null, + "owner": "input-output-hk", + "repo": "cardano-shell", + "rev": "8e0d6f5548c1c13a9edfeb56e4ea27c0a6d10948", + "sha256": "07yjl3agiacmy5irxxmrmm7ffpnvz84zgrylnp3wmcrn417pc5fq", + "type": "tarball", + "url": "https://github.com/input-output-hk/cardano-shell/archive/8e0d6f5548c1c13a9edfeb56e4ea27c0a6d10948.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "ghc-8.6.5-iohk": { "branch": "release/8.6.5-iohk", "deepClone": true, @@ -26,6 +39,19 @@ "url": "https://github.com/input-output-hk/ghc.git", "url_template": "https://github.com///archive/.tar.gz" }, + "hpc-coveralls": { + "branch": "master", + "builtin": false, + "description": "coveralls.io support for haskell code coverage with hpc", + "homepage": "http://hackage.haskell.org/package/hpc-coveralls", + "owner": "sevanspowell", + "repo": "hpc-coveralls", + "rev": "14df0f7d229f4cd2e79f8eabb1a740097fdfa430", + "sha256": "02dmcmqc845s7sdgdnk3xxn7l6jj8faa7547b4cii9mgv09arspj", + "type": "tarball", + "url": "https://github.com/sevanspowell/hpc-coveralls/archive/14df0f7d229f4cd2e79f8eabb1a740097fdfa430.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "niv": { "branch": "master", "builtin": false, diff --git a/test/coverage-golden/default.nix b/test/coverage-golden/default.nix new file mode 100644 index 00000000..70adfb5a --- /dev/null +++ b/test/coverage-golden/default.nix @@ -0,0 +1,50 @@ +{ stdenv, fetchFromGitHub, recurseIntoAttrs, runCommand, testSrc, compiler-nix-name, buildPackages, sources }: + +with stdenv.lib; + +let + # Using buildPackages.buildPackages here because buildPackages.git + # is built with a cross compiler version of gdb (needed by python). + # So buildPackages.buildPackages.git is more likely to be in the cache. + inherit (buildPackages.buildPackages) jq git; + + hpc-coveralls-exes = (buildPackages.haskell-nix.project' { + compiler-nix-name = "ghc865"; # TODO use `inherit compiler-nix-name;` once it is working with 8.8 and 8.10 + src = sources.hpc-coveralls; + }).hsPkgs.hpc-coveralls.components.exes; + + exampleProjectSrc = sources.cardano-shell; + + exampleProject = import "${exampleProjectSrc}" { config = { haskellNix = { coverage = true; }; }; }; + exampleCoverageReport = exampleProject.cardanoShellHaskellPackages.projectCoverageReport; + +in recurseIntoAttrs ({ + run = stdenv.mkDerivation { + name = "coverage-golden-test"; + + nativeBuildInputs = [ git jq hpc-coveralls-exes.hpc-coveralls hpc-coveralls-exes.run-cabal-test ]; + buildCommand = '' + ######################################################################## + # Test that the coverage reports haven't materially changed + + TRAVIS=1 TRAVIS_JOB_ID=1 hpc-coveralls all \ + --package-dir ${exampleProjectSrc}/cardano-shell/ \ + --package-dir ${exampleProjectSrc}/cardano-launcher \ + --hpc-dir ${exampleCoverageReport}/share/hpc/vanilla \ + --coverage-mode StrictlyFullLines \ + --dont-send + + # Format JSON for better diffing error messages and remove + # references to Nix path, our golden coverage file contains no + # Nix paths + cat ./travis-ci-1.json | jq -S | sed "s;${exampleProjectSrc}/;;g" > ./actual.json + + # Fail if doesn't match golden, i.e. our coverage has changed + diff ${./golden.json} ./actual.json + + touch $out + ''; + + meta.platforms = platforms.all; + }; +}) diff --git a/test/coverage-golden/golden.json b/test/coverage-golden/golden.json new file mode 100644 index 00000000..af052ccb --- /dev/null +++ b/test/coverage-golden/golden.json @@ -0,0 +1,2069 @@ +{ + "service_job_id": "1", + "service_name": "travis-ci", + "source_files": [ + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + 0, + 0, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + 0, + null, + 0, + null, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + 0, + 1, + null, + 0, + 1, + null, + 0, + null, + 0, + 1, + null, + 1, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + null, + null, + null, + 1, + 0, + null + ], + "name": "cardano-launcher/src/Cardano/Shell/CLI.hs", + "source_digest": "b9e41a4fca61e6b7b0c5b9b194c7ec74" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + 0, + null, + null, + null, + 0, + null, + null, + null, + 0, + 0, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + 1, + null, + null, + 0, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + null, + null, + null, + 0, + 0, + 0, + null, + null, + 0, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + null, + 0, + 0, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + 0, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + null + ], + "name": "cardano-launcher/src/Cardano/Shell/Configuration.hs", + "source_digest": "4dd11901f7f3443c1da61c9f48b9638e" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + null, + null, + null, + null, + 1, + null, + null, + 0, + null, + null, + null, + null, + 1, + 1, + 1, + 1 + ], + "name": "cardano-launcher/src/Cardano/Shell/Environment.hs", + "source_digest": "8f79decd0d4f6d2cc403efac6dfe0c5f" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + 0, + null, + null, + null, + null, + 1, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 1, + 1, + 0, + null, + null, + 1, + 1, + null, + null, + 1, + 0, + null, + null, + 1, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + null, + 1, + 1, + null, + 1, + 1, + 1, + 1, + null, + 1, + 1, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 0, + null, + null, + 1, + 1, + 1, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 0, + null, + null, + 0, + null, + 0, + null, + 1, + null, + 1, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + 0, + null, + null, + null, + 0, + 0, + 0, + null, + 0, + null, + null, + 0, + null, + 0, + null, + null, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + 0, + null, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + 0, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + 0, + 0, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + 0, + null, + null, + null, + null, + 0, + null, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + 0, + null, + 0, + null, + 0, + 0, + null, + null, + 0, + null, + 0, + 0, + 0, + null, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 0, + null, + 0, + null, + null, + 0, + null, + null, + 0, + null, + null, + 0, + null + ], + "name": "cardano-launcher/src/Cardano/Shell/Launcher.hs", + "source_digest": "4c5ed3d1086f1563907e0ab857992926" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 0, + 1, + null, + null, + null, + null, + null, + 1, + 0, + 0, + 0, + null, + null + ], + "name": "cardano-launcher/src/Cardano/Shell/Launcher/Types.hs", + "source_digest": "c13a2fdef971017159a191b8a8fc1984" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 0, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + 0, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + 1, + 1, + 1, + 1, + 0, + null, + 1, + null, + 1, + 1, + null, + 1, + 1, + null, + 1, + null, + 0, + 0, + null, + null, + null, + 1, + null, + 1, + 1, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 1, + null, + 1, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0, + null, + null, + 1, + 1, + 0, + 1, + 0, + null, + null, + 1, + 1, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 0, + null, + null, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + 1, + null, + null, + 1, + 1, + 1, + 1, + 1, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 1, + null, + 0, + null, + null, + 1, + 1, + null, + null, + 1, + null, + null, + 0, + 0, + 0, + null, + null, + 1 + ], + "name": "cardano-launcher/src/Cardano/Shell/Template.hs", + "source_digest": "8ed410823bb76c849cb1d68b2c338fdc" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + 1, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + 0, + 1, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + null, + 1, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 1, + null, + null, + null, + 1, + 1, + null, + null, + null, + 1, + 0, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 1, + 1, + null, + null, + 1, + 1, + null, + null, + null, + 1, + null, + 0, + null, + null, + null, + 0, + null, + 1, + 1, + null, + null, + 0, + null, + null, + null, + 1, + 0, + 0, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + 0, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + 1, + 1, + 1, + null, + 0, + 0, + null, + 1, + null, + 1, + null, + null, + null, + 1, + null, + 1, + 0, + 1, + null, + 1, + null, + 1, + 0, + 1, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ], + "name": "cardano-launcher/src/Cardano/Shell/Update/Lib.hs", + "source_digest": "f2c59081d819c717e9ebd81744d13445" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 1, + 1, + 1, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + 0, + null, + null, + 1, + 1, + 1, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + null, + null, + 0, + 0, + null, + null, + 0, + null, + null, + 0, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + null, + 0, + 0, + null, + null, + null, + 1, + 1, + 1, + 0, + null, + 0, + 0, + 0, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + null, + null, + 1, + 0, + 0, + 0, + 0, + null, + null, + null, + 1, + 1, + 1, + null, + null, + 0, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + null, + 1, + null, + null, + 1, + 1, + 1, + null, + 1, + 1, + 1, + null, + null, + 1, + 1, + 1, + 1, + null, + null, + 1, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + 1, + 0, + 1, + null, + null, + 1, + 1, + null, + null, + null, + null, + 1, + 0, + 0, + 0, + 0, + null, + null, + null, + 1, + 1, + 1, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + null, + 1, + 1, + null, + null, + null, + null, + 1, + 1, + null, + null, + null, + null, + null, + 1, + null, + 1, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + null, + 1, + 0, + null, + null, + 1, + null, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + null, + 1, + null, + 1, + 1, + null, + null, + 1, + 1, + null, + null, + null, + null, + null, + 1, + null, + 1, + 1, + null, + 1, + 1, + null, + 1, + null, + null, + null, + null, + 1, + 1, + null, + 1, + 1, + null, + 1, + 1, + null, + 1, + null, + null, + null, + null, + 1, + 1, + 1, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + null, + 1, + null, + null, + 1, + null, + null, + 1, + null, + null, + 1, + 1, + 1, + 1, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + null, + null, + null, + null, + 1, + 0, + null, + null, + null, + 1, + 0, + null, + null, + null, + 1, + 0, + null, + null, + null, + 1, + 0, + null, + null, + null, + 1, + 0 + ], + "name": "cardano-shell/src/Cardano/Shell/NodeIPC/Lib.hs", + "source_digest": "fa9891027aa304903e21266794047ce6" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + 1, + null, + null, + null, + null, + 1, + null, + null, + 1, + 1, + 1, + null, + null, + 0, + 1, + null, + null, + 0, + 0, + 0, + 0, + 0, + 0, + null, + null, + 1, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + 1, + 0, + 1, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + null, + 1, + 1, + 1, + null, + null, + 0, + 0, + 0, + null, + null, + 0, + 0, + 0, + null, + null + ], + "name": "cardano-shell/src/Cardano/Shell/NodeIPC/Message.hs", + "source_digest": "6606e99fa2eb594ea4b966753a471e30" + }, + { + "coverage": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + null, + 0, + null, + 0, + null, + 0, + 0, + null, + 0, + 0, + null, + 0, + null, + 0, + null, + null, + null, + null, + null, + null, + null, + 0, + null, + null, + 0, + 0, + null, + null, + 0, + 0, + null, + null, + null, + 0, + 0, + null, + null, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + 0, + 0, + 0, + 0, + 0, + null, + 0, + 0, + 0, + 0, + null, + null, + null, + null, + 0, + null, + 0, + 0, + 0, + 0, + null + ], + "name": "cardano-shell/src/Cardano/Shell/NodeIPC/ServerExample.hs", + "source_digest": "a6e42e140ae0b20912179bcef22889a2" + } + ] +} diff --git a/test/default.nix b/test/default.nix index 7de232b4..0ab0d5ef 100644 --- a/test/default.nix +++ b/test/default.nix @@ -193,6 +193,7 @@ let } // lib.optionalAttrs (!stdenv.hostPlatform.isGhcjs) { # These do not work on ghcjs because it needs zlib. coverage = callTest ./coverage { inherit compiler-nix-name; }; + coverage-golden = callTest ./coverage-golden { inherit compiler-nix-name;}; coverage-no-libs = callTest ./coverage-no-libs { inherit compiler-nix-name; }; snapshots = callTest ./snapshots {}; } // lib.optionalAttrs (!stdenv.hostPlatform.isGhcjs && compiler-nix-name != "ghc8101" && compiler-nix-name != "ghc8102" && compiler-nix-name != "ghc810220201118" ) {