diff --git a/tests/expectations/parser/parser/expression/literal/string.leo.out b/tests/expectations/parser/parser/expression/literal/string.leo.out index b6e44d99a1..11e9e9e3d6 100644 --- a/tests/expectations/parser/parser/expression/literal/string.leo.out +++ b/tests/expectations/parser/parser/expression/literal/string.leo.out @@ -1,268 +1,31 @@ --- -namespace: ParseExpression +namespace: Token expectation: Pass outputs: - - Value: - String: - - - Scalar: 115 - - Scalar: 116 - - Scalar: 114 - - Scalar: 105 - - Scalar: 110 - - Scalar: 103 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 9 - path: "" - content: "\"string\"" - - Value: - String: - - - Scalar: 97 - - Scalar: 110 - - Scalar: 111 - - Scalar: 116 - - Scalar: 104 - - Scalar: 101 - - Scalar: 114 - - Scalar: 32 - - Scalar: 123 - - Scalar: 32 - - Scalar: 125 - - Scalar: 32 - - Scalar: 115 - - Scalar: 116 - - Scalar: 114 - - Scalar: 105 - - Scalar: 110 - - Scalar: 103 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 21 - path: "" - content: "\"another { } string\"" - - Value: - String: - - - Scalar: 123 - - Scalar: 32 - - Scalar: 93 - - Scalar: 32 - - Scalar: 91 - - Scalar: 32 - - Scalar: 59 - - Scalar: 32 - - Scalar: 97 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 12 - path: "" - content: "\"{ ] [ ; a\"" - - Value: - String: - - - Scalar: 4090 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 10 - path: "" - content: "\"\\u{FFA}\"" - - Value: - String: - - - Scalar: 719610 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 12 - path: "" - content: "\"\\u{afafa}\"" - - Value: - String: - - - Scalar: 44975 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 11 - path: "" - content: "\"\\u{afaf}\"" - - Value: - String: - - - Scalar: 2810 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 10 - path: "" - content: "\"\\u{afa}\"" - - Value: - String: - - - Scalar: 175 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 9 - path: "" - content: "\"\\u{af}\"" - - Value: - String: - - - Scalar: 10 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 8 - path: "" - content: "\"\\u{a}\"" - - Value: - String: - - - Scalar: 10 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 7 - path: "" - content: "\"\\x0A\"" - - Value: - String: - - - Scalar: 127 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 7 - path: "" - content: "\"\\x7F\"" - - Value: - String: - - - Scalar: 97 - - Scalar: 97 - - Scalar: 32 - - Scalar: 92 - - Scalar: 32 - - Scalar: 34 - - Scalar: 32 - - Scalar: 39 - - Scalar: 32 - - Scalar: 10 - - Scalar: 32 - - Scalar: 97 - - Scalar: 97 - - Scalar: 32 - - Scalar: 9 - - Scalar: 32 - - Scalar: 13 - - Scalar: 32 - - Scalar: 32 - - Scalar: 0 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 30 - path: "" - content: "\"aa \\\\ \\\" \\' \\n aa \\t \\r \\0\"" - - Value: - String: - - - Scalar: 116 - - Scalar: 101 - - Scalar: 115 - - Scalar: 116 - - Scalar: 32 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 15 - path: "" - content: "\"test 😒€\"" - - Value: - String: - - [] - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 15 - path: "" - content: "\"😭😂😘\"" - - Value: - String: - - [] - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 10 - path: "" - content: "\"✋🏿\"" - - Value: - String: - - [] - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 7 - path: "" - content: "\"🦀\"" - - Value: - String: - - [] - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 6 - path: "" - content: "\"￿\"" - - Value: - String: - - [] - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 12 - path: "" - content: "\"���\"" - - Value: - String: - - - Scalar: 65288 - - Scalar: 65299 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 17 - path: "" - content: "\"(>3<)三\"" - - Value: - String: - - - Scalar: 98 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 4 - path: "" - content: "\"b\" // TODO reenabe once #1682 is closed \"(⑅∫°ਊ°)∫\"" - - Value: - String: - - - Scalar: 98 - - span: - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 4 - path: "" - content: "\"b\" // TODO reenabe once #1682 is closed \"🦀°1\"" + - "'\"string\"' @ 1:1-9" + - "'\"another { } string\"' @ 1:1-21" + - "'\"{ ] [ ; a\"' @ 1:1-12" + - "'\"࿺\"' @ 1:1-10" + - "'\"򯫺\"' @ 1:1-12" + - "'\"꾯\"' @ 1:1-11" + - "'\"ૺ\"' @ 1:1-10" + - "'\"¯\"' @ 1:1-9" + - "'\"\n\"' @ 1:1-8" + - "'\"\n\"' @ 1:1-7" + - "'\"\u007f\"' @ 1:1-7" + - "'\"aa \\ \" ' \n aa \t \r \u0000\"' @ 1:1-30" + - "'\"test \"' @ 1:1-15" + - "'\"\"' @ 1:1-15" + - "'\"\"' @ 1:1-10" + - "'\"\"' @ 1:1-7" + - "'\"\"' @ 1:1-6" + - "'\"\"' @ 1:1-12" + - "'\"(3\"' @ 1:1-17" + - "'\"ヽಠ\"' @ 1:1-26" + - "'\"(╯\"' @ 1:1-33" + - "'\"┬ノ ゜゜\"' @ 1:1-29" + - "'\"( ͜͡͡\"' @ 1:1-20" + - "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"ᕙ(▀̿ĺ̯▀̿ ̿)ᕗ\"' @ 1:5-69" + - "'\"♥-_-]\"' @ 1:1-20" + - "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"(⑅∫°ਊ°)∫\"' @ 1:5-62" + - "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"🦀°1\"' @ 1:5-51" diff --git a/tests/parser/expression/literal/string.leo b/tests/parser/expression/literal/string.leo index 7c230e7bd6..fc9711e444 100644 --- a/tests/parser/expression/literal/string.leo +++ b/tests/parser/expression/literal/string.leo @@ -1,5 +1,5 @@ /* -namespace: ParseExpression +namespace: Token expectation: Pass */ @@ -32,5 +32,11 @@ expectation: Pass "���" "(>3<)三" +"ヽ༼ ಠ益ಠ ༽ノ" +"(╯°□°)╯︵ ┻━┻" +"┬─┬ ノ( ゜-゜ノ)" +"( ͡° ͜ʖ ͡°)" +"b" // TODO reenabe once #1682 is closed "ᕙ(▀̿ĺ̯▀̿ ̿)ᕗ" +"♥╣[-_-]╠♥" "b" // TODO reenabe once #1682 is closed "(⑅∫°ਊ°)∫" "b" // TODO reenabe once #1682 is closed "🦀°1" \ No newline at end of file diff --git a/tests/test-framework/src/error.rs b/tests/test-framework/src/error.rs index fbf8f8dadd..d540942eab 100644 --- a/tests/test-framework/src/error.rs +++ b/tests/test-framework/src/error.rs @@ -27,19 +27,28 @@ pub struct TestFailure { #[derive(Debug)] pub enum TestError { + Panicked { + test: String, + index: usize, + error: String, + }, UnexpectedOutput { + test: String, index: usize, expected: Value, output: Value, }, PassedAndShouldntHave { + test: String, index: usize, }, FailedAndShouldntHave { + test: String, index: usize, error: String, }, UnexpectedError { + test: String, index: usize, expected: String, output: String, @@ -50,30 +59,64 @@ pub enum TestError { impl fmt::Display for TestError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let format_test = |test: &str| -> String { + if test.len() > 50 { + String::new() + } else { + format!("\n\n{}\n\n", test) + } + }; match self { + TestError::Panicked { test, index, error } => { + write!( + f, + "test #{}: {}encountered a rust panic:\n{}", + index + 1, + format_test(test), + error + ) + } TestError::UnexpectedOutput { + test, index, expected, output, } => { write!( f, - "test #{} expected\n{}\ngot\n{}", + "test #{}: {}expected\n{}\ngot\n{}", index + 1, + format_test(test), serde_yaml::to_string(&expected).expect("serialization failed"), serde_yaml::to_string(&output).expect("serialization failed") ) } - TestError::PassedAndShouldntHave { index } => write!(f, "test #{} passed and shouldn't have", index + 1), - TestError::FailedAndShouldntHave { index, error } => { - write!(f, "test #{} failed and shouldn't have:\n{}", index + 1, error) + TestError::PassedAndShouldntHave { test, index } => { + write!(f, "test #{}: {}passed and shouldn't have", index + 1, format_test(test)) + } + TestError::FailedAndShouldntHave { test, index, error } => { + write!( + f, + "test #{}: {}failed and shouldn't have:\n{}", + index + 1, + format_test(test), + error + ) } TestError::UnexpectedError { + test, expected, output, index, } => { - write!(f, "test #{} expected error\n{}\ngot\n{}", index + 1, expected, output) + write!( + f, + "test #{}: {}expected error\n{}\ngot\n{}", + index + 1, + format_test(test), + expected, + output + ) } TestError::MismatchedTestExpectationLength => write!(f, "invalid number of test expectations"), TestError::MissingTestConfig => write!(f, "missing test config"), @@ -82,18 +125,25 @@ impl fmt::Display for TestError { } pub fn emit_errors( - output: Result<&Value, &str>, + test: &str, + output: &Result, String>, mode: &TestExpectationMode, expected_output: Option, test_index: usize, ) -> Option { match (output, mode) { - (Ok(output), TestExpectationMode::Pass) => { + (Err(e), _) => Some(TestError::Panicked { + test: test.to_string(), + index: test_index, + error: e.to_string(), + }), + (Ok(Ok(output)), TestExpectationMode::Pass) => { // passed and should have if let Some(expected_output) = expected_output.as_ref() { if output != expected_output { // invalid output return Some(TestError::UnexpectedOutput { + test: test.to_string(), index: test_index, expected: expected_output.clone(), output: output.clone(), @@ -102,18 +152,23 @@ pub fn emit_errors( } None } - (Ok(_tokens), TestExpectationMode::Fail) => Some(TestError::PassedAndShouldntHave { index: test_index }), - (Err(err), TestExpectationMode::Pass) => Some(TestError::FailedAndShouldntHave { + (Ok(Ok(_tokens)), TestExpectationMode::Fail) => Some(TestError::PassedAndShouldntHave { + test: test.to_string(), + index: test_index, + }), + (Ok(Err(err)), TestExpectationMode::Pass) => Some(TestError::FailedAndShouldntHave { + test: test.to_string(), error: err.to_string(), index: test_index, }), - (Err(err), TestExpectationMode::Fail) => { + (Ok(Err(err)), TestExpectationMode::Fail) => { let expected_output: Option = expected_output.map(|x| serde_yaml::from_value(x).expect("test expectation deserialize failed")); if let Some(expected_output) = expected_output.as_deref() { if err != expected_output { // invalid output return Some(TestError::UnexpectedError { + test: test.to_string(), expected: expected_output.to_string(), output: err.to_string(), index: test_index, diff --git a/tests/test-framework/src/runner.rs b/tests/test-framework/src/runner.rs index 3865e2a2e3..522a33a34b 100644 --- a/tests/test-framework/src/runner.rs +++ b/tests/test-framework/src/runner.rs @@ -16,8 +16,12 @@ use serde_yaml::Value; use std::{ + any::Any, collections::BTreeMap, + panic::{self, RefUnwindSafe, UnwindSafe}, path::{Path, PathBuf}, + sync::{Arc, Mutex}, + thread, }; use crate::{error::*, fetch::find_tests, output::TestExpectation, test::*}; @@ -36,7 +40,7 @@ pub struct Test { pub config: BTreeMap, } -pub trait Namespace { +pub trait Namespace: UnwindSafe + RefUnwindSafe { fn parse_type(&self) -> ParseType; fn run_test(&self, test: Test) -> Result; @@ -46,6 +50,36 @@ pub trait Runner { fn resolve_namespace(&self, name: &str) -> Option>; } +fn set_hook() -> Arc>> { + let panic_buf = Arc::new(Mutex::new(None)); + let thread_id = thread::current().id(); + panic::set_hook({ + let panic_buf = panic_buf.clone(); + Box::new(move |e| { + if thread::current().id() == thread_id { + *panic_buf.lock().unwrap() = Some(e.to_string()); + } else { + println!("{}", e) + } + }) + }); + panic_buf +} + +fn take_hook( + output: Result, Box>, + panic_buf: Arc>>, +) -> Result, String> { + output.map_err(|_| { + panic_buf + .lock() + .unwrap() + .take() + .expect("failed to get panic message") + .clone() + }) +} + pub fn run_tests(runner: &T, expectation_category: &str) { std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files std::env::set_var("LEO_TESTFRAMEWORK", "true"); @@ -149,24 +183,24 @@ pub fn run_tests(runner: &T, expectation_category: &str) { for (i, test) in tests.into_iter().enumerate() { let expected_output = expected_output.as_mut().and_then(|x| x.next()).cloned(); println!("running test {} @ '{}'", test_name, path.to_str().unwrap()); - let output = namespace.run_test(Test { - name: test_name.clone(), - content: test.clone(), - path: path.into(), - config: config.extra.clone(), + let panic_buf = set_hook(); + let leo_output = panic::catch_unwind(|| { + namespace.run_test(Test { + name: test_name.clone(), + content: test.clone(), + path: path.into(), + config: config.extra.clone(), + }) }); - if let Some(error) = emit_errors( - output.as_ref().map_err(|x| &**x), - &config.expectation, - expected_output, - i, - ) { + let output = take_hook(leo_output, panic_buf); + if let Some(error) = emit_errors(&test, &output, &config.expectation, expected_output, i) { fail_tests += 1; errors.push(error); } else { pass_tests += 1; new_outputs.push( output + .unwrap() .as_ref() .map(|x| serde_yaml::to_value(x).expect("serialization failed")) .unwrap_or_else(|e| Value::String(e.clone())),