diff --git a/Cargo.lock b/Cargo.lock index 5b32f65ac2..e3ad834687 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1485,6 +1485,7 @@ dependencies = [ "leo-compiler", "leo-imports", "leo-parser", + "regex", "serde", "serde_json", "serde_yaml", diff --git a/test-framework/Cargo.toml b/test-framework/Cargo.toml index f32f3fe07a..169193230f 100644 --- a/test-framework/Cargo.toml +++ b/test-framework/Cargo.toml @@ -55,3 +55,7 @@ version = "1.5.2" [dependencies.structopt] version = "0.3" + +# List of dependencies for errcov +[dependencies.regex] +version = "1.5" \ No newline at end of file diff --git a/test-framework/src/bin/errcov.rs b/test-framework/src/bin/errcov.rs new file mode 100644 index 0000000000..b3a9a88cf3 --- /dev/null +++ b/test-framework/src/bin/errcov.rs @@ -0,0 +1,117 @@ +// Copyright (C) 2019-2021 Aleo Systems Inc. +// This file is part of the Leo library. + +// The Leo library is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// The Leo library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with the Leo library. If not, see . + +use leo_asg::Asg; +use leo_ast::AstPass; +use leo_compiler::{compiler::thread_leaked_context, TypeInferencePhase}; +use leo_imports::ImportParser; +use leo_test_framework::{ + fetch::find_tests, + output::TestExpectation, + test::{extract_test_config, TestExpectationMode as Expectation}, +}; + +use regex::Regex; +use std::{collections::HashSet, error::Error, fs, path::PathBuf}; +use structopt::{clap::AppSettings, StructOpt}; + +#[derive(StructOpt)] +#[structopt(name = "error-coverage", author = "The Aleo Team ", setting = AppSettings::ColoredHelp)] +struct Opt { + #[structopt(short, long, help = "Path to the output file", parse(from_os_str))] + output: PathBuf, +} + +fn main() { + handle_error(run_with_args(Opt::from_args())); +} + +fn run_with_args(opt: Opt) -> Result<(), Box> { + // Variable that stores all the tests. + let mut tests = Vec::new(); + let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_dir.push("../tests/"); + + let mut expectation_dir = test_dir.clone(); + expectation_dir.push("expectations"); + + find_tests(&test_dir, &mut tests); + + // Store all covered error codes + let mut codes = HashSet::new(); + let re = Regex::new(r"Error \[(?P.*)\]:.*").unwrap(); + + for (path, content) in tests.into_iter() { + if let Some(config) = extract_test_config(&content) { + // Skip passing tests. + if config.expectation == Expectation::Pass { + continue; + } + + let mut expectation_path = expectation_dir.clone(); + + let path = PathBuf::from(path); + let relative_path = path.strip_prefix(&test_dir).expect("path error for test"); + + let mut relative_expectation_path = relative_path.to_str().unwrap().to_string(); + relative_expectation_path += ".out"; + + // Add expectation category + if relative_expectation_path.starts_with("compiler") { + expectation_path.push("compiler"); + } else { + expectation_path.push("parser"); + } + expectation_path.push(&relative_expectation_path); + + if expectation_path.exists() { + let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file"); + let expectation: TestExpectation = + serde_yaml::from_str(&raw).expect("invalid yaml in expectations file"); + + for value in expectation.outputs { + if let serde_yaml::Value::String(message) = value { + if let Some(caps) = re.captures(&message) { + if let Some(code) = caps.name("code") { + codes.insert(code.as_str().to_string()); + } + } + } + } + } + } + } + + let mut sorted: Vec<_> = codes.iter().collect(); + sorted.sort(); + + println!("Found the following error codes"); + for code in sorted { + println!("{}", code) + } + + Ok(()) +} + +fn handle_error(res: Result<(), Box>) { + match res { + Ok(_) => (), + Err(err) => { + eprintln!("Error: {}", err); + std::process::exit(1); + } + } +}