diff --git a/Cargo.lock b/Cargo.lock index 4fe601ff45..f79f18c1de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1483,8 +1483,10 @@ dependencies = [ "leo-ast", "leo-ast-passes", "leo-compiler", + "leo-errors", "leo-imports", "leo-parser", + "regex", "serde", "serde_json", "serde_yaml", diff --git a/errors/src/common/macros.rs b/errors/src/common/macros.rs index 9dfa00f67c..54551c9b53 100644 --- a/errors/src/common/macros.rs +++ b/errors/src/common/macros.rs @@ -19,7 +19,13 @@ /// with a unique error code. #[macro_export] macro_rules! create_errors { - (@step $_code:expr,) => {}; + (@step $code:expr,) => { + #[inline(always)] + // Returns the number of unique exit codes that this error type can take on. + pub fn num_exit_codes() -> i32 { + $code + } + }; ($(#[$error_type_docs:meta])* $error_type:ident, exit_code_mask: $exit_code_mask:expr, error_code_prefix: $error_code_prefix:expr, $($(#[$docs:meta])* @$formatted_or_backtraced_list:ident $names:ident { args: ($($arg_names:ident: $arg_types:ty$(,)?)*), msg: $messages:expr, help: $helps:expr, })*) => { #[allow(unused_imports)] // Allow unused for errors that only use formatted or backtraced errors. use crate::{BacktracedError, FormattedError, LeoErrorCode, Span}; @@ -63,6 +69,7 @@ macro_rules! create_errors { } } + // Steps over the list of functions with an initial error code of 0. impl $error_type { create_errors!(@step 0i32, $(($(#[$docs])* $formatted_or_backtraced_list, $names($($arg_names: $arg_types,)*), $messages, $helps),)*); @@ -112,5 +119,4 @@ macro_rules! create_errors { // Steps the error code value by one and calls on the rest of the functions. create_errors!(@step $code + 1i32, $(($(#[$docs])* $formatted_or_backtraced_tail, $names($($tail_arg_names: $tail_arg_types,)*), $messages, $helps),)*); }; - } diff --git a/test-framework/Cargo.toml b/test-framework/Cargo.toml index f32f3fe07a..bea2f59c5c 100644 --- a/test-framework/Cargo.toml +++ b/test-framework/Cargo.toml @@ -55,3 +55,11 @@ version = "1.5.2" [dependencies.structopt] version = "0.3" + +# List of dependencies for errcov +[dependencies.leo-errors] +path = "../errors" +version = "1.5.3" + +[dependencies.regex] +version = "1.5" \ No newline at end of file diff --git a/test-framework/src/bin/errcov.rs b/test-framework/src/bin/errcov.rs new file mode 100644 index 0000000000..2d18415e71 --- /dev/null +++ b/test-framework/src/bin/errcov.rs @@ -0,0 +1,269 @@ +// Copyright (C) 2019-2021 Aleo Systems Inc. +// This file is part of the Leo library. + +// The Leo library is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// The Leo library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with the Leo library. If not, see . + +use leo_errors::{ + AsgError, AstError, CliError, CompilerError, ImportError, LeoErrorCode, PackageError, ParserError, StateError, +}; +use leo_test_framework::{ + fetch::find_tests, + output::TestExpectation, + test::{extract_test_config, TestExpectationMode as Expectation}, +}; + +use regex::Regex; +use serde_yaml::Value; +use std::collections::{BTreeMap, HashSet}; +use std::{error::Error, fs, io, path::PathBuf}; +use structopt::{clap::AppSettings, StructOpt}; + +#[derive(StructOpt)] +#[structopt(name = "error-coverage", author = "The Aleo Team ", setting = AppSettings::ColoredHelp)] +struct Opt { + #[structopt( + short, + long, + help = "Path to the output file, defaults to stdout.", + parse(from_os_str) + )] + output: Option, +} + +fn main() { + handle_error(run_with_args(Opt::from_args())); +} + +fn run_with_args(opt: Opt) -> Result<(), Box> { + // Variable that stores all the tests. + let mut tests = Vec::new(); + let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_dir.push("../tests/"); + + let mut expectation_dir = test_dir.clone(); + expectation_dir.push("expectations"); + + find_tests(&test_dir, &mut tests); + + // Store all covered error codes + let mut found_codes = BTreeMap::new(); + let re = Regex::new(r"Error \[(?P.*)\]:.*").unwrap(); + + for (path, content) in tests.into_iter() { + if let Some(config) = extract_test_config(&content) { + // Skip passing tests. + if config.expectation == Expectation::Pass { + continue; + } + + let mut expectation_path = expectation_dir.clone(); + + let path = PathBuf::from(path); + let relative_path = path.strip_prefix(&test_dir).expect("path error for test"); + + let mut relative_expectation_path = relative_path.to_str().unwrap().to_string(); + relative_expectation_path += ".out"; + + // Add expectation category + if relative_expectation_path.starts_with("compiler") { + expectation_path.push("compiler"); + } else { + expectation_path.push("parser"); + } + expectation_path.push(&relative_expectation_path); + + if expectation_path.exists() { + let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file"); + let expectation: TestExpectation = + serde_yaml::from_str(&raw).expect("invalid yaml in expectations file"); + + for value in expectation.outputs { + if let serde_yaml::Value::String(message) = value { + if let Some(caps) = re.captures(&message) { + if let Some(code) = caps.name("code") { + let files = found_codes + .entry(code.as_str().to_string()) + .or_insert_with(HashSet::new); + let path = expectation_path + .strip_prefix(test_dir.clone()) + .expect("invalid prefix for expectation path"); + files.insert(PathBuf::from(path)); + } + } + } + } + } + } + } + + // Collect all defined error codes. + let mut all_codes = HashSet::new(); + collect_error_codes( + &mut all_codes, + AsgError::error_type(), + AsgError::code_identifier(), + AsgError::exit_code_mask(), + AsgError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + AstError::error_type(), + AstError::code_identifier(), + AstError::exit_code_mask(), + AstError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + CliError::error_type(), + CliError::code_identifier(), + CliError::exit_code_mask(), + CliError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + CompilerError::error_type(), + CompilerError::code_identifier(), + CompilerError::exit_code_mask(), + CompilerError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + ImportError::error_type(), + ImportError::code_identifier(), + ImportError::exit_code_mask(), + ImportError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + PackageError::error_type(), + PackageError::code_identifier(), + PackageError::exit_code_mask(), + PackageError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + ParserError::error_type(), + ParserError::code_identifier(), + ParserError::exit_code_mask(), + ParserError::num_exit_codes(), + ); + collect_error_codes( + &mut all_codes, + StateError::error_type(), + StateError::code_identifier(), + StateError::exit_code_mask(), + StateError::num_exit_codes(), + ); + + // Repackage data into values compatible with serde_yaml + let mut covered_errors = serde_yaml::Mapping::new(); + let mut unknown_errors = serde_yaml::Mapping::new(); + + for (code, paths) in found_codes.iter() { + let mut yaml_paths = Vec::with_capacity(paths.len()); + for path in paths { + yaml_paths.push(path.to_str().unwrap()); + } + yaml_paths.sort_unstable(); + let yaml_paths = yaml_paths.iter().map(|s| Value::String(s.to_string())).collect(); + + if all_codes.contains(code) { + covered_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths)); + } else { + unknown_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths)); + } + all_codes.remove(code); + } + + let mut codes: Vec = all_codes.drain().collect(); + codes.sort(); + + let mut uncovered_errors = Vec::new(); + for code in codes { + uncovered_errors.push(Value::String(code)) + } + + let mut uncovered_information = serde_yaml::Mapping::new(); + uncovered_information.insert( + Value::String(String::from("count")), + Value::Number(serde_yaml::Number::from(uncovered_errors.len())), + ); + uncovered_information.insert(Value::String(String::from("codes")), Value::Sequence(uncovered_errors)); + + let mut covered_information = serde_yaml::Mapping::new(); + covered_information.insert( + Value::String(String::from("count")), + Value::Number(serde_yaml::Number::from(covered_errors.len())), + ); + covered_information.insert(Value::String(String::from("codes")), Value::Mapping(covered_errors)); + + let mut unknown_information = serde_yaml::Mapping::new(); + unknown_information.insert( + Value::String(String::from("count")), + Value::Number(serde_yaml::Number::from(unknown_errors.len())), + ); + unknown_information.insert(Value::String(String::from("codes")), Value::Mapping(unknown_errors)); + + let mut results = serde_yaml::Mapping::new(); + results.insert( + Value::String(String::from("uncovered")), + Value::Mapping(uncovered_information), + ); + + results.insert( + Value::String(String::from("covered")), + Value::Mapping(covered_information), + ); + results.insert( + Value::String(String::from("unknown")), + Value::Mapping(unknown_information), + ); + + // Output error coverage results + if let Some(pathbuf) = opt.output { + let file = fs::File::create(pathbuf).expect("error creating output file"); + serde_yaml::to_writer(file, &results).expect("serialization failed for error coverage report"); + } else { + serde_yaml::to_writer(io::stdout(), &results).expect("serialization failed for error coverage report"); + } + + Ok(()) +} + +fn collect_error_codes( + codes: &mut HashSet, + error_type: String, + code_identifier: i8, + exit_code_mask: i32, + num_exit_codes: i32, +) { + for exit_code in 0..num_exit_codes { + codes.insert(format!( + "E{}{:0>3}{:0>4}", + error_type, + code_identifier, + exit_code_mask + exit_code, + )); + } +} + +fn handle_error(res: Result<(), Box>) { + match res { + Ok(_) => (), + Err(err) => { + eprintln!("Error: {}", err); + std::process::exit(1); + } + } +}