diff --git a/Cargo.lock b/Cargo.lock
index 4fe601ff45..f79f18c1de 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1483,8 +1483,10 @@ dependencies = [
"leo-ast",
"leo-ast-passes",
"leo-compiler",
+ "leo-errors",
"leo-imports",
"leo-parser",
+ "regex",
"serde",
"serde_json",
"serde_yaml",
diff --git a/errors/src/common/macros.rs b/errors/src/common/macros.rs
index 9dfa00f67c..54551c9b53 100644
--- a/errors/src/common/macros.rs
+++ b/errors/src/common/macros.rs
@@ -19,7 +19,13 @@
/// with a unique error code.
#[macro_export]
macro_rules! create_errors {
- (@step $_code:expr,) => {};
+ (@step $code:expr,) => {
+ #[inline(always)]
+ // Returns the number of unique exit codes that this error type can take on.
+ pub fn num_exit_codes() -> i32 {
+ $code
+ }
+ };
($(#[$error_type_docs:meta])* $error_type:ident, exit_code_mask: $exit_code_mask:expr, error_code_prefix: $error_code_prefix:expr, $($(#[$docs:meta])* @$formatted_or_backtraced_list:ident $names:ident { args: ($($arg_names:ident: $arg_types:ty$(,)?)*), msg: $messages:expr, help: $helps:expr, })*) => {
#[allow(unused_imports)] // Allow unused for errors that only use formatted or backtraced errors.
use crate::{BacktracedError, FormattedError, LeoErrorCode, Span};
@@ -63,6 +69,7 @@ macro_rules! create_errors {
}
}
+
// Steps over the list of functions with an initial error code of 0.
impl $error_type {
create_errors!(@step 0i32, $(($(#[$docs])* $formatted_or_backtraced_list, $names($($arg_names: $arg_types,)*), $messages, $helps),)*);
@@ -112,5 +119,4 @@ macro_rules! create_errors {
// Steps the error code value by one and calls on the rest of the functions.
create_errors!(@step $code + 1i32, $(($(#[$docs])* $formatted_or_backtraced_tail, $names($($tail_arg_names: $tail_arg_types,)*), $messages, $helps),)*);
};
-
}
diff --git a/test-framework/Cargo.toml b/test-framework/Cargo.toml
index f32f3fe07a..bea2f59c5c 100644
--- a/test-framework/Cargo.toml
+++ b/test-framework/Cargo.toml
@@ -55,3 +55,11 @@ version = "1.5.2"
[dependencies.structopt]
version = "0.3"
+
+# List of dependencies for errcov
+[dependencies.leo-errors]
+path = "../errors"
+version = "1.5.3"
+
+[dependencies.regex]
+version = "1.5"
\ No newline at end of file
diff --git a/test-framework/src/bin/errcov.rs b/test-framework/src/bin/errcov.rs
new file mode 100644
index 0000000000..2d18415e71
--- /dev/null
+++ b/test-framework/src/bin/errcov.rs
@@ -0,0 +1,269 @@
+// Copyright (C) 2019-2021 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see .
+
+use leo_errors::{
+ AsgError, AstError, CliError, CompilerError, ImportError, LeoErrorCode, PackageError, ParserError, StateError,
+};
+use leo_test_framework::{
+ fetch::find_tests,
+ output::TestExpectation,
+ test::{extract_test_config, TestExpectationMode as Expectation},
+};
+
+use regex::Regex;
+use serde_yaml::Value;
+use std::collections::{BTreeMap, HashSet};
+use std::{error::Error, fs, io, path::PathBuf};
+use structopt::{clap::AppSettings, StructOpt};
+
+#[derive(StructOpt)]
+#[structopt(name = "error-coverage", author = "The Aleo Team ", setting = AppSettings::ColoredHelp)]
+struct Opt {
+ #[structopt(
+ short,
+ long,
+ help = "Path to the output file, defaults to stdout.",
+ parse(from_os_str)
+ )]
+ output: Option,
+}
+
+fn main() {
+ handle_error(run_with_args(Opt::from_args()));
+}
+
+fn run_with_args(opt: Opt) -> Result<(), Box> {
+ // Variable that stores all the tests.
+ let mut tests = Vec::new();
+ let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+ test_dir.push("../tests/");
+
+ let mut expectation_dir = test_dir.clone();
+ expectation_dir.push("expectations");
+
+ find_tests(&test_dir, &mut tests);
+
+ // Store all covered error codes
+ let mut found_codes = BTreeMap::new();
+ let re = Regex::new(r"Error \[(?P.*)\]:.*").unwrap();
+
+ for (path, content) in tests.into_iter() {
+ if let Some(config) = extract_test_config(&content) {
+ // Skip passing tests.
+ if config.expectation == Expectation::Pass {
+ continue;
+ }
+
+ let mut expectation_path = expectation_dir.clone();
+
+ let path = PathBuf::from(path);
+ let relative_path = path.strip_prefix(&test_dir).expect("path error for test");
+
+ let mut relative_expectation_path = relative_path.to_str().unwrap().to_string();
+ relative_expectation_path += ".out";
+
+ // Add expectation category
+ if relative_expectation_path.starts_with("compiler") {
+ expectation_path.push("compiler");
+ } else {
+ expectation_path.push("parser");
+ }
+ expectation_path.push(&relative_expectation_path);
+
+ if expectation_path.exists() {
+ let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file");
+ let expectation: TestExpectation =
+ serde_yaml::from_str(&raw).expect("invalid yaml in expectations file");
+
+ for value in expectation.outputs {
+ if let serde_yaml::Value::String(message) = value {
+ if let Some(caps) = re.captures(&message) {
+ if let Some(code) = caps.name("code") {
+ let files = found_codes
+ .entry(code.as_str().to_string())
+ .or_insert_with(HashSet::new);
+ let path = expectation_path
+ .strip_prefix(test_dir.clone())
+ .expect("invalid prefix for expectation path");
+ files.insert(PathBuf::from(path));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ // Collect all defined error codes.
+ let mut all_codes = HashSet::new();
+ collect_error_codes(
+ &mut all_codes,
+ AsgError::error_type(),
+ AsgError::code_identifier(),
+ AsgError::exit_code_mask(),
+ AsgError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ AstError::error_type(),
+ AstError::code_identifier(),
+ AstError::exit_code_mask(),
+ AstError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ CliError::error_type(),
+ CliError::code_identifier(),
+ CliError::exit_code_mask(),
+ CliError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ CompilerError::error_type(),
+ CompilerError::code_identifier(),
+ CompilerError::exit_code_mask(),
+ CompilerError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ ImportError::error_type(),
+ ImportError::code_identifier(),
+ ImportError::exit_code_mask(),
+ ImportError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ PackageError::error_type(),
+ PackageError::code_identifier(),
+ PackageError::exit_code_mask(),
+ PackageError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ ParserError::error_type(),
+ ParserError::code_identifier(),
+ ParserError::exit_code_mask(),
+ ParserError::num_exit_codes(),
+ );
+ collect_error_codes(
+ &mut all_codes,
+ StateError::error_type(),
+ StateError::code_identifier(),
+ StateError::exit_code_mask(),
+ StateError::num_exit_codes(),
+ );
+
+ // Repackage data into values compatible with serde_yaml
+ let mut covered_errors = serde_yaml::Mapping::new();
+ let mut unknown_errors = serde_yaml::Mapping::new();
+
+ for (code, paths) in found_codes.iter() {
+ let mut yaml_paths = Vec::with_capacity(paths.len());
+ for path in paths {
+ yaml_paths.push(path.to_str().unwrap());
+ }
+ yaml_paths.sort_unstable();
+ let yaml_paths = yaml_paths.iter().map(|s| Value::String(s.to_string())).collect();
+
+ if all_codes.contains(code) {
+ covered_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths));
+ } else {
+ unknown_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths));
+ }
+ all_codes.remove(code);
+ }
+
+ let mut codes: Vec = all_codes.drain().collect();
+ codes.sort();
+
+ let mut uncovered_errors = Vec::new();
+ for code in codes {
+ uncovered_errors.push(Value::String(code))
+ }
+
+ let mut uncovered_information = serde_yaml::Mapping::new();
+ uncovered_information.insert(
+ Value::String(String::from("count")),
+ Value::Number(serde_yaml::Number::from(uncovered_errors.len())),
+ );
+ uncovered_information.insert(Value::String(String::from("codes")), Value::Sequence(uncovered_errors));
+
+ let mut covered_information = serde_yaml::Mapping::new();
+ covered_information.insert(
+ Value::String(String::from("count")),
+ Value::Number(serde_yaml::Number::from(covered_errors.len())),
+ );
+ covered_information.insert(Value::String(String::from("codes")), Value::Mapping(covered_errors));
+
+ let mut unknown_information = serde_yaml::Mapping::new();
+ unknown_information.insert(
+ Value::String(String::from("count")),
+ Value::Number(serde_yaml::Number::from(unknown_errors.len())),
+ );
+ unknown_information.insert(Value::String(String::from("codes")), Value::Mapping(unknown_errors));
+
+ let mut results = serde_yaml::Mapping::new();
+ results.insert(
+ Value::String(String::from("uncovered")),
+ Value::Mapping(uncovered_information),
+ );
+
+ results.insert(
+ Value::String(String::from("covered")),
+ Value::Mapping(covered_information),
+ );
+ results.insert(
+ Value::String(String::from("unknown")),
+ Value::Mapping(unknown_information),
+ );
+
+ // Output error coverage results
+ if let Some(pathbuf) = opt.output {
+ let file = fs::File::create(pathbuf).expect("error creating output file");
+ serde_yaml::to_writer(file, &results).expect("serialization failed for error coverage report");
+ } else {
+ serde_yaml::to_writer(io::stdout(), &results).expect("serialization failed for error coverage report");
+ }
+
+ Ok(())
+}
+
+fn collect_error_codes(
+ codes: &mut HashSet,
+ error_type: String,
+ code_identifier: i8,
+ exit_code_mask: i32,
+ num_exit_codes: i32,
+) {
+ for exit_code in 0..num_exit_codes {
+ codes.insert(format!(
+ "E{}{:0>3}{:0>4}",
+ error_type,
+ code_identifier,
+ exit_code_mask + exit_code,
+ ));
+ }
+}
+
+fn handle_error(res: Result<(), Box>) {
+ match res {
+ Ok(_) => (),
+ Err(err) => {
+ eprintln!("Error: {}", err);
+ std::process::exit(1);
+ }
+ }
+}