ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-11-23 15:15:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

improved test runner

Browse Source
This commit is contained in:
0rphon 2022-03-15 16:56:54 -07:00
parent cc63b7e524
commit d25eb79594
4 changed files with 146 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

293
tests/expectations/parser/parser/expression/literal/string.leo.out
View File

@ -1,268 +1,31 @@
---
namespace: ParseExpression
namespace: Token
expectation: Pass
outputs:
- Value:
String:
- - Scalar: 115
- Scalar: 116
- Scalar: 114
- Scalar: 105
- Scalar: 110
- Scalar: 103
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 9
path: ""
content: "\"string\""
- Value:
String:
- - Scalar: 97
- Scalar: 110
- Scalar: 111
- Scalar: 116
- Scalar: 104
- Scalar: 101
- Scalar: 114
- Scalar: 32
- Scalar: 123
- Scalar: 32
- Scalar: 125
- Scalar: 32
- Scalar: 115
- Scalar: 116
- Scalar: 114
- Scalar: 105
- Scalar: 110
- Scalar: 103
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 21
path: ""
content: "\"another { } string\""
- Value:
String:
- - Scalar: 123
- Scalar: 32
- Scalar: 93
- Scalar: 32
- Scalar: 91
- Scalar: 32
- Scalar: 59
- Scalar: 32
- Scalar: 97
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 12
path: ""
content: "\"{ ] [ ; a\""
- Value:
String:
- - Scalar: 4090
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 10
path: ""
content: "\"\\u{FFA}\""
- Value:
String:
- - Scalar: 719610
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 12
path: ""
content: "\"\\u{afafa}\""
- Value:
String:
- - Scalar: 44975
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 11
path: ""
content: "\"\\u{afaf}\""
- Value:
String:
- - Scalar: 2810
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 10
path: ""
content: "\"\\u{afa}\""
- Value:
String:
- - Scalar: 175
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 9
path: ""
content: "\"\\u{af}\""
- Value:
String:
- - Scalar: 10
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 8
path: ""
content: "\"\\u{a}\""
- Value:
String:
- - Scalar: 10
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 7
path: ""
content: "\"\\x0A\""
- Value:
String:
- - Scalar: 127
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 7
path: ""
content: "\"\\x7F\""
- Value:
String:
- - Scalar: 97
- Scalar: 97
- Scalar: 32
- Scalar: 92
- Scalar: 32
- Scalar: 34
- Scalar: 32
- Scalar: 39
- Scalar: 32
- Scalar: 10
- Scalar: 32
- Scalar: 97
- Scalar: 97
- Scalar: 32
- Scalar: 9
- Scalar: 32
- Scalar: 13
- Scalar: 32
- Scalar: 32
- Scalar: 0
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 30
path: ""
content: "\"aa \\\\ \\\" \\' \\n aa \\t \\r \\0\""
- Value:
String:
- - Scalar: 116
- Scalar: 101
- Scalar: 115
- Scalar: 116
- Scalar: 32
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 15
path: ""
content: "\"test 😒€\""
- Value:
String:
- []
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 15
path: ""
content: "\"😭😂😘\""
- Value:
String:
- []
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 10
path: ""
content: "\"✋🏿\""
- Value:
String:
- []
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 7
path: ""
content: "\"🦀\""
- Value:
String:
- []
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 6
path: ""
content: "\"￿\""
- Value:
String:
- []
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 12
path: ""
content: "\"<22><><EFBFBD>\""
- Value:
String:
- - Scalar: 65288
- Scalar: 65299
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 17
path: ""
content: "\"（>３<）三\""
- Value:
String:
- - Scalar: 98
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 4
path: ""
content: "\"b\" // TODO reenabe once #1682 is closed \"(⑅∫°ਊ°)∫\""
- Value:
String:
- - Scalar: 98
- span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 4
path: ""
content: "\"b\" // TODO reenabe once #1682 is closed \"🦀°1\""
- "'\"string\"' @ 1:1-9"
- "'\"another { } string\"' @ 1:1-21"
- "'\"{ ] [ ; a\"' @ 1:1-12"
- "'\"࿺\"' @ 1:1-10"
- "'\"򯫺\"' @ 1:1-12"
- "'\"꾯\"' @ 1:1-11"
- "'\"ૺ\"' @ 1:1-10"
- "'\"¯\"' @ 1:1-9"
- "'\"\n\"' @ 1:1-8"
- "'\"\n\"' @ 1:1-7"
- "'\"\u007f\"' @ 1:1-7"
- "'\"aa \\ \" ' \n aa \t \r \u0000\"' @ 1:1-30"
- "'\"test \"' @ 1:1-15"
- "'\"\"' @ 1:1-15"
- "'\"\"' @ 1:1-10"
- "'\"\"' @ 1:1-7"
- "'\"\"' @ 1:1-6"
- "'\"\"' @ 1:1-12"
- "'\"（３\"' @ 1:1-17"
- "'\"ヽಠ\"' @ 1:1-26"
- "'\"(╯\"' @ 1:1-33"
- "'\"┬ノ ゜゜\"' @ 1:1-29"
- "'\"( ͜͡͡\"' @ 1:1-20"
- "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"ᕙ(▀̿ĺ̯▀̿ ̿)ᕗ\"' @ 1:5-69"
- "'\"♥-_-]\"' @ 1:1-20"
- "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"(⑅∫°ਊ°)∫\"' @ 1:5-62"
- "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"🦀°1\"' @ 1:5-51"

8
tests/parser/expression/literal/string.leo
View File

@ -1,5 +1,5 @@
/*
namespace: ParseExpression
namespace: Token
expectation: Pass
*/
@ -32,5 +32,11 @@ expectation: Pass
"<22><><EFBFBD>"
"（>３<）三"
"ヽ༼ ಠ益ಠ ༽ﾉ"
"(╯°□°）╯︵ ┻━┻"
"┬─┬ ノ( ゜-゜ノ)"
"( ͡° ͜ʖ ͡°)"
"b" // TODO reenabe once #1682 is closed "ᕙ(▀̿ĺ̯▀̿ ̿)ᕗ"
"♥╣[-_-]╠♥"
"b" // TODO reenabe once #1682 is closed "(⑅∫°ਊ°)∫"
"b" // TODO reenabe once #1682 is closed "🦀°1"

75
tests/test-framework/src/error.rs
View File

@ -27,19 +27,28 @@ pub struct TestFailure {
#[derive(Debug)]
pub enum TestError {
Panicked {
test: String,
index: usize,
error: String,
},
UnexpectedOutput {
test: String,
index: usize,
expected: Value,
output: Value,
},
PassedAndShouldntHave {
test: String,
index: usize,
},
FailedAndShouldntHave {
test: String,
index: usize,
error: String,
},
UnexpectedError {
test: String,
index: usize,
expected: String,
output: String,
@ -50,30 +59,64 @@ pub enum TestError {
impl fmt::Display for TestError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let format_test = |test: &str| -> String {
if test.len() > 50 {
String::new()
} else {
format!("\n\n{}\n\n", test)
}
};
match self {
TestError::Panicked { test, index, error } => {
write!(
f,
"test #{}: {}encountered a rust panic:\n{}",
index + 1,
format_test(test),
error
)
}
TestError::UnexpectedOutput {
test,
index,
expected,
output,
} => {
write!(
f,
"test #{} expected\n{}\ngot\n{}",
"test #{}: {}expected\n{}\ngot\n{}",
index + 1,
format_test(test),
serde_yaml::to_string(&expected).expect("serialization failed"),
serde_yaml::to_string(&output).expect("serialization failed")
)
}
TestError::PassedAndShouldntHave { index } => write!(f, "test #{} passed and shouldn't have", index + 1),
TestError::FailedAndShouldntHave { index, error } => {
write!(f, "test #{} failed and shouldn't have:\n{}", index + 1, error)
TestError::PassedAndShouldntHave { test, index } => {
write!(f, "test #{}: {}passed and shouldn't have", index + 1, format_test(test))
}
TestError::FailedAndShouldntHave { test, index, error } => {
write!(
f,
"test #{}: {}failed and shouldn't have:\n{}",
index + 1,
format_test(test),
error
)
}
TestError::UnexpectedError {
test,
expected,
output,
index,
} => {
write!(f, "test #{} expected error\n{}\ngot\n{}", index + 1, expected, output)
write!(
f,
"test #{}: {}expected error\n{}\ngot\n{}",
index + 1,
format_test(test),
expected,
output
)
}
TestError::MismatchedTestExpectationLength => write!(f, "invalid number of test expectations"),
TestError::MissingTestConfig => write!(f, "missing test config"),
@ -82,18 +125,25 @@ impl fmt::Display for TestError {
}
pub fn emit_errors(
output: Result<&Value, &str>,
test: &str,
output: &Result<Result<Value, String>, String>,
mode: &TestExpectationMode,
expected_output: Option<Value>,
test_index: usize,
) -> Option<TestError> {
match (output, mode) {
(Ok(output), TestExpectationMode::Pass) => {
(Err(e), _) => Some(TestError::Panicked {
test: test.to_string(),
index: test_index,
error: e.to_string(),
}),
(Ok(Ok(output)), TestExpectationMode::Pass) => {
// passed and should have
if let Some(expected_output) = expected_output.as_ref() {
if output != expected_output {
// invalid output
return Some(TestError::UnexpectedOutput {
test: test.to_string(),
index: test_index,
expected: expected_output.clone(),
output: output.clone(),
@ -102,18 +152,23 @@ pub fn emit_errors(
}
None
}
(Ok(_tokens), TestExpectationMode::Fail) => Some(TestError::PassedAndShouldntHave { index: test_index }),
(Err(err), TestExpectationMode::Pass) => Some(TestError::FailedAndShouldntHave {
(Ok(Ok(_tokens)), TestExpectationMode::Fail) => Some(TestError::PassedAndShouldntHave {
test: test.to_string(),
index: test_index,
}),
(Ok(Err(err)), TestExpectationMode::Pass) => Some(TestError::FailedAndShouldntHave {
test: test.to_string(),
error: err.to_string(),
index: test_index,
}),
(Err(err), TestExpectationMode::Fail) => {
(Ok(Err(err)), TestExpectationMode::Fail) => {
let expected_output: Option<String> =
expected_output.map(|x| serde_yaml::from_value(x).expect("test expectation deserialize failed"));
if let Some(expected_output) = expected_output.as_deref() {
if err != expected_output {
// invalid output
return Some(TestError::UnexpectedError {
test: test.to_string(),
expected: expected_output.to_string(),
output: err.to_string(),
index: test_index,

58
tests/test-framework/src/runner.rs
View File

@ -16,8 +16,12 @@
use serde_yaml::Value;
use std::{
any::Any,
collections::BTreeMap,
panic::{self, RefUnwindSafe, UnwindSafe},
path::{Path, PathBuf},
sync::{Arc, Mutex},
thread,
};
use crate::{error::*, fetch::find_tests, output::TestExpectation, test::*};
@ -36,7 +40,7 @@ pub struct Test {
pub config: BTreeMap<String, Value>,
}
pub trait Namespace {
pub trait Namespace: UnwindSafe + RefUnwindSafe {
fn parse_type(&self) -> ParseType;
fn run_test(&self, test: Test) -> Result<Value, String>;
@ -46,6 +50,36 @@ pub trait Runner {
fn resolve_namespace(&self, name: &str) -> Option<Box<dyn Namespace>>;
}
fn set_hook() -> Arc<Mutex<Option<String>>> {
let panic_buf = Arc::new(Mutex::new(None));
let thread_id = thread::current().id();
panic::set_hook({
let panic_buf = panic_buf.clone();
Box::new(move |e| {
if thread::current().id() == thread_id {
*panic_buf.lock().unwrap() = Some(e.to_string());
} else {
println!("{}", e)
}
})
});
panic_buf
}
fn take_hook(
output: Result<Result<Value, String>, Box<dyn Any + Send>>,
panic_buf: Arc<Mutex<Option<String>>>,
) -> Result<Result<Value, String>, String> {
output.map_err(|_| {
panic_buf
.lock()
.unwrap()
.take()
.expect("failed to get panic message")
.clone()
})
}
pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files
std::env::set_var("LEO_TESTFRAMEWORK", "true");
@ -149,24 +183,24 @@ pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
for (i, test) in tests.into_iter().enumerate() {
let expected_output = expected_output.as_mut().and_then(|x| x.next()).cloned();
println!("running test {} @ '{}'", test_name, path.to_str().unwrap());
let output = namespace.run_test(Test {
name: test_name.clone(),
content: test.clone(),
path: path.into(),
config: config.extra.clone(),
let panic_buf = set_hook();
let leo_output = panic::catch_unwind(|| {
namespace.run_test(Test {
name: test_name.clone(),
content: test.clone(),
path: path.into(),
config: config.extra.clone(),
})
});
if let Some(error) = emit_errors(
output.as_ref().map_err(|x| &**x),
&config.expectation,
expected_output,
i,
) {
let output = take_hook(leo_output, panic_buf);
if let Some(error) = emit_errors(&test, &output, &config.expectation, expected_output, i) {
fail_tests += 1;
errors.push(error);
} else {
pass_tests += 1;
new_outputs.push(
output
.unwrap()
.as_ref()
.map(|x| serde_yaml::to_value(x).expect("serialization failed"))
.unwrap_or_else(|e| Value::String(e.clone())),