From 0bc0a72d8aa82562e06621609844ca6257fa07b7 Mon Sep 17 00:00:00 2001 From: Swapnil Kumbhar Date: Tue, 11 Oct 2022 04:54:31 -0700 Subject: [PATCH] Code coverage for Rust (#1263) * feat: Bumped Rust to stable-2022-04-07 (1.60.0) * feat: Added coverage tests and report generation * feat: Added workflow for Rust Coverage * feat: Bumped Rust to 1.60.0 in Workflows * test: Testing for CI * ci: Added name and fixed path for Rust Coverage * revert: "test: Testing for CI" This reverts commit 049a7b108822a4f468d35d20f432bd5002d5d461. * fix: Added .pub-cache to PATH, change incremental builds to zero * test: Testing for CI * ci: Installing protobuf compiler in CI * revert: "test: Testing for CI" This reverts commit 049a7b108822a4f468d35d20f432bd5002d5d461. * ci: Added environment setup for dart * test: Testing for CI * feat: Added task to check if 'grcov' is installed * ci: Added steps to install grcov * test: Testing for CI * ci: Fixed name for grcov installation step * feat: Added cargo binaries to PATH in tests * revert: "test: Testing for CI" This reverts commit 24e425db66fa160528be310bb7a6337f5c8ea06e. * revert: "test: Testing for CI" This reverts commit a5de816bce647b815921700540c652023f788a58. * ci: Fixed a very silly failing case * test: Created a test rust file for cov test --- .github/workflows/ci.yaml | 2 +- .github/workflows/dart_lint.yml | 2 +- .github/workflows/dart_test.yml | 2 +- .github/workflows/release.yml | 6 +- .github/workflows/rust_coverage.yml | 71 +++++++++ .github/workflows/rust_lint.yml | 2 +- .github/workflows/rust_test.yml | 4 +- frontend/rust-lib/covtest.rs | 0 frontend/rust-lib/rust-toolchain.toml | 2 +- frontend/scripts/makefile/tests.toml | 200 ++++++++++++++++++++++++++ shared-lib/rust-toolchain.toml | 2 +- 11 files changed, 282 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/rust_coverage.yml create mode 100644 frontend/rust-lib/covtest.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e456dda9aa..e3ca2c3a76 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,7 +29,7 @@ jobs: - id: rust_toolchain uses: actions-rs/toolchain@v1 with: - toolchain: "stable-2022-01-20" + toolchain: "stable-2022-04-07" - id: flutter uses: subosito/flutter-action@v2 diff --git a/.github/workflows/dart_lint.yml b/.github/workflows/dart_lint.yml index 30f65cc752..6ce8630b8d 100644 --- a/.github/workflows/dart_lint.yml +++ b/.github/workflows/dart_lint.yml @@ -34,7 +34,7 @@ jobs: channel: "stable" - uses: actions-rs/toolchain@v1 with: - toolchain: "stable-2022-01-20" + toolchain: "stable-2022-04-07" - name: Cache Cargo id: cache-cargo diff --git a/.github/workflows/dart_test.yml b/.github/workflows/dart_test.yml index 9171fb6dea..902a861239 100644 --- a/.github/workflows/dart_test.yml +++ b/.github/workflows/dart_test.yml @@ -24,7 +24,7 @@ jobs: - uses: actions-rs/toolchain@v1 with: - toolchain: "stable-2022-01-20" + toolchain: "stable-2022-04-07" - uses: subosito/flutter-action@v2 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bbfc131fd..c9c1cb00f8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,7 @@ jobs: - name: Setup environment - Rust and Cargo uses: actions-rs/toolchain@v1 with: - toolchain: 'stable-2022-01-20' + toolchain: 'stable-2022-04-07' - name: Setup environment - Flutter uses: subosito/flutter-action@v2 @@ -94,7 +94,7 @@ jobs: - name: Setup environment - Rust and Cargo uses: actions-rs/toolchain@v1 with: - toolchain: 'stable-2022-01-20' + toolchain: 'stable-2022-04-07' - name: Setup environment - Flutter uses: subosito/flutter-action@v2 @@ -143,7 +143,7 @@ jobs: - name: Setup environment - Rust and Cargo uses: actions-rs/toolchain@v1 with: - toolchain: 'stable-2022-01-20' + toolchain: 'stable-2022-04-07' - name: Setup environment - Flutter uses: subosito/flutter-action@v2 diff --git a/.github/workflows/rust_coverage.yml b/.github/workflows/rust_coverage.yml new file mode 100644 index 0000000000..b63d0c2202 --- /dev/null +++ b/.github/workflows/rust_coverage.yml @@ -0,0 +1,71 @@ +name: Rust coverage tests + +on: + push: + branches: + - "main" + paths: + - "frontend/rust-lib/**" + - "shared-lib/**" + + pull_request: + branches: + - "main" + paths: + - "frontend/rust-lib/**" + - "shared-lib/**" + +env: + CARGO_TERM_COLOR: always + + +jobs: + test-coverage: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - id: rust_toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: 'stable-2022-04-07' + + - name: Cache Cargo + uses: actions/cache@v2 + with: + path: | + ~/.cargo + key: ${{ runner.os }}-cargo-${{ steps.rust_toolchain.outputs.rustc_hash }}-${{ hashFiles('./frontend/rust-lib/Cargo.toml') }} + + - name: Cache Rust + uses: actions/cache@v2 + with: + path: | + frontend/rust-lib/target + shared-lib/target + key: ${{ runner.os }}-rust-rust-lib-share-lib-${{ steps.rust_toolchain.outputs.rustc_hash }}-${{ hashFiles('./frontend/rust-lib/Cargo.toml') }} + + - name: Setup Environment + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo wget -qO /etc/apt/trusted.gpg.d/dart_linux_signing_key.asc https://dl-ssl.google.com/linux/linux_signing_key.pub + sudo wget -qO /etc/apt/sources.list.d/dart_stable.list https://storage.googleapis.com/download.dartlang.org/linux/debian/dart_stable.list + sudo apt-get update + sudo apt-get install -y dart curl build-essential libsqlite3-dev libssl-dev clang cmake ninja-build pkg-config libgtk-3-dev + sudo apt-get install keybinder-3.0 + elif [ "$RUNNER_OS" == "macOS" ]; then + echo 'do nothing' + fi + shell: bash + + - name: Install cargo-make, grcov and llvm-tools-preview + working-directory: frontend + run: | + cargo install cargo-make + cargo install grcov + rustup component add llvm-tools-preview + + - name: Run Coverage tests and generate LCOV report + working-directory: frontend + run: cargo make get_ci_test_coverage diff --git a/.github/workflows/rust_lint.yml b/.github/workflows/rust_lint.yml index 435b1388f2..2221ad02a0 100644 --- a/.github/workflows/rust_lint.yml +++ b/.github/workflows/rust_lint.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: - toolchain: 'stable-2022-01-20' + toolchain: 'stable-2022-04-07' override: true - uses: subosito/flutter-action@v1 with: diff --git a/.github/workflows/rust_test.yml b/.github/workflows/rust_test.yml index 4fb2a2ec0a..3a42f9074f 100644 --- a/.github/workflows/rust_test.yml +++ b/.github/workflows/rust_test.yml @@ -28,7 +28,7 @@ jobs: - id: rust_toolchain uses: actions-rs/toolchain@v1 with: - toolchain: 'stable-2022-01-20' + toolchain: 'stable-2022-04-07' - name: Cache Cargo uses: actions/cache@v2 @@ -55,4 +55,4 @@ jobs: - name: Run shared-lib tests working-directory: shared-lib - run: RUST_LOG=info cargo test --no-default-features \ No newline at end of file + run: RUST_LOG=info cargo test --no-default-features diff --git a/frontend/rust-lib/covtest.rs b/frontend/rust-lib/covtest.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frontend/rust-lib/rust-toolchain.toml b/frontend/rust-lib/rust-toolchain.toml index 48a0631f2e..a6b0571dd2 100644 --- a/frontend/rust-lib/rust-toolchain.toml +++ b/frontend/rust-lib/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "stable-2022-01-20" \ No newline at end of file +channel = "stable-2022-04-07" diff --git a/frontend/scripts/makefile/tests.toml b/frontend/scripts/makefile/tests.toml index b8082f651b..02c680e71c 100644 --- a/frontend/scripts/makefile/tests.toml +++ b/frontend/scripts/makefile/tests.toml @@ -16,3 +16,203 @@ cd ../shared-lib RUST_LOG=info cargo test --no-default-features ''' +[tasks.check_grcov] +description = "Check if `grcov` is installled" +script_runner = "@shell" +script = ''' +export PATH=$PATH:"$HOME/.cargo/bin/" +if command -v grcov > /dev/null; then + echo "Found 'grcov' executable." +else + echo "[!] Could not find 'grcov' executable." + echo "[!] Please install 'grcov' by running 'cargo install grcov'." + echo "[!] You may also need to install 'llvm-tools-preview' using 'rustup component add llvm-tools-preview'." + echo "[!] If installed, check if 'grcov' is in PATH." + echo "[!] Exiting..." + exit -1 +fi +''' + +[tasks.clean_profraw_files] +description = "Cleans profraw files that are created by `cargo test`" +script_runner = "@duckscript" +script = [ + """ + rust_lib_profs = glob_array ./rust-lib/**/*.profraw + for prof in ${rust_lib_profs} + full_path = canonicalize ${prof} + rm ${full_path} + end + + shared_lib_profs = glob_array ../shared-lib/**/*.profraw + for prof in ${shared_lib_profs} + full_path = canonicalize ${prof} + rm ${full_path} + end + + """ +] + +[tasks.run_rustlib_coverage_tests] +description = "Run tests with coverage instrumentation" +script_runner = "@shell" +script = [ + """ + echo --- Running coverage tests --- + + # Install Protobuf compiler + cargo make install_protobuf_compiler + export PATH="$PATH":"$HOME/.pub-cache/bin" + export PATH="$PATH":"$HOME/.cargo/bin/" + + cd rust-lib/ + + CARGO_INCREMENTAL=0 \ + RUSTFLAGS='-C instrument-coverage' \ + LLVM_PROFILE_FILE='prof-%p-%m.profraw' \ + cargo test --no-default-features --features="sync" + + """ +] + +[tasks.run_sharedlib_coverage_tests] +description = "Run tests with coverage instrumentation" +script_runner = "@shell" +script = [ + """ + echo --- Running coverage tests --- + + # Install Protobuf compiler + cargo make install_protobuf_compiler + export PATH="$PATH":"$HOME/.pub-cache/bin" + export PATH="$PATH":"$HOME/.cargo/bin/" + + cd ../shared-lib + + CARGO_INCREMENTAL=0 \ + RUSTFLAGS='-C instrument-coverage' \ + LLVM_PROFILE_FILE='prof-%p-%m.profraw' \ + cargo test --no-default-features + + """ +] + +[tasks.get_rustlib_grcov_report] +description = "Get `grcov` HTML report for test coverage for rust-lib" +script_runner = "@shell" +script = [ + """ + echo --- Getting 'grcov' results for 'rust-lib' --- + cd rust-lib/ + + grcov . \ + --binary-path target/debug/deps \ + --source-dir . \ + --output-type html \ + --branch \ + --ignore-not-existing \ + --log-level WARN \ + --output-path target/coverage-html + + echo "--- Done! Generated HTML report under 'target/coverage-html' for rustlib." + """ +] + +[tasks.get_sharedlib_grcov_report] +description = "Get `grcov` HTML report for test coverage shared-lib" +script_runner = "@shell" +script = [ + """ + echo --- Getting 'grcov' results 'shared-lib' --- + cd ../shared-lib + + grcov . \ + --binary-path target/debug/deps \ + --source-dir . \ + --output-type html \ + --branch \ + --ignore-not-existing \ + --log-level WARN \ + --output-path target/coverage-html + + echo "--- Done! Generated HTML report under 'target/coverage-html' for sharedlib." + """ +] + +[tasks.get_grcov_report] +description = "Get `grcov` HTML report for test coverage" +run_task = { name = [ + "get_rustlib_grcov_report", + "get_sharedlib_grcov_report" +], parallel = true } + +[tasks.get_sharedlib_lcov_report] +description = "Generates `lcov` report for `shared-lib`" +script_runner = "@shell" +script = [ + """ + echo Getting 'lcov' results for 'shared-lib' + + cd ../shared-lib + + grcov . \ + --binary-path target/debug/deps \ + --source-dir . \ + --output-type lcov \ + --branch \ + --ignore-not-existing \ + --log-level WARN \ + --output-path target/coverage.lcov + + echo "--- Done! Generated 'target/coverage.lcov' sharedlib." + """ +] + +[tasks.get_rustlib_lcov_report] +description = "Generates `lcov` report for `rust-lib`" +script_runner = "@shell" +script = [ + """ + echo Getting 'lcov' results for 'rust-lib' + + cd rust-lib/ + + grcov . \ + --binary-path target/debug/deps \ + --source-dir . \ + --output-type lcov \ + --branch \ + --ignore-not-existing \ + --log-level WARN \ + --output-path target/coverage.lcov + + echo "--- Done! Generated 'target/coverage.lcov' for rustlib." + """ +] + +[tasks.get_lcov_report] +description = "Get `lcov` reports for test coverage" +run_task = { name = [ + "get_sharedlib_lcov_report", + "get_rustlib_lcov_report" +], parallel = true } + +[tasks.get_ci_test_coverage] +description = "Get LCOV coverage reports for CI" +run_task = { name = [ + "check_grcov", + "run_rustlib_coverage_tests", + "run_sharedlib_coverage_tests", + "get_lcov_report", + "clean_profraw_files" + ]} + +[tasks.get_test_coverage] +description = "Get human readable test coverage reports" +run_task = { name = [ + "check_grcov", + "run_rustlib_coverage_tests", + "run_sharedlib_coverage_tests", + "get_grcov_report", + "clean_profraw_files" + ]} diff --git a/shared-lib/rust-toolchain.toml b/shared-lib/rust-toolchain.toml index 48a0631f2e..a6b0571dd2 100644 --- a/shared-lib/rust-toolchain.toml +++ b/shared-lib/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "stable-2022-01-20" \ No newline at end of file +channel = "stable-2022-04-07"