improved test runner

tests/expectations/parser/parser/expression/literal/string.leo.out

@@ -1,268 +1,31 @@
 ---
-namespace: ParseExpression
+namespace: Token
 expectation: Pass
 outputs:
-  - Value:
+  - "'\"string\"' @ 1:1-9"
-      String:
+  - "'\"another { } string\"' @ 1:1-21"
-        - - Scalar: 115
+  - "'\"{ ] [ ; a\"' @ 1:1-12"
-          - Scalar: 116
+  - "'\"࿺\"' @ 1:1-10"
-          - Scalar: 114
+  - "'\"򯫺\"' @ 1:1-12"
-          - Scalar: 105
+  - "'\"꾯\"' @ 1:1-11"
-          - Scalar: 110
+  - "'\"ૺ\"' @ 1:1-10"
-          - Scalar: 103
+  - "'\"¯\"' @ 1:1-9"
-        - span:
+  - "'\"\n\"' @ 1:1-8"
-            line_start: 1
+  - "'\"\n\"' @ 1:1-7"
-            line_stop: 1
+  - "'\"\u007f\"' @ 1:1-7"
-            col_start: 1
+  - "'\"aa \\ \" ' \n aa \t \r  \u0000\"' @ 1:1-30"
-            col_stop: 9
+  - "'\"test \"' @ 1:1-15"
-            path: ""
+  - "'\"\"' @ 1:1-15"
-            content: "\"string\""
+  - "'\"\"' @ 1:1-10"
-  - Value:
+  - "'\"\"' @ 1:1-7"
-      String:
+  - "'\"\"' @ 1:1-6"
-        - - Scalar: 97
+  - "'\"\"' @ 1:1-12"
-          - Scalar: 110
+  - "'\"（３\"' @ 1:1-17"
-          - Scalar: 111
+  - "'\"ヽಠ\"' @ 1:1-26"
-          - Scalar: 116
+  - "'\"(╯\"' @ 1:1-33"
-          - Scalar: 104
+  - "'\"┬ノ ゜゜\"' @ 1:1-29"
-          - Scalar: 101
+  - "'\"( ͜͡͡\"' @ 1:1-20"
-          - Scalar: 114
+  - "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"ᕙ(▀̿ĺ̯▀̿ ̿)ᕗ\"' @ 1:5-69"
-          - Scalar: 32
+  - "'\"♥-_-]\"' @ 1:1-20"
-          - Scalar: 123
+  - "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"(⑅∫°ਊ°)∫\"' @ 1:5-62"
-          - Scalar: 32
+  - "'\"b\"' @ 1:1-4,'// TODO reenabe once #1682 is closed \"🦀°1\"' @ 1:5-51"
-          - Scalar: 125
-          - Scalar: 32
-          - Scalar: 115
-          - Scalar: 116
-          - Scalar: 114
-          - Scalar: 105
-          - Scalar: 110
-          - Scalar: 103
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 21
-            path: ""
-            content: "\"another { } string\""
-  - Value:
-      String:
-        - - Scalar: 123
-          - Scalar: 32
-          - Scalar: 93
-          - Scalar: 32
-          - Scalar: 91
-          - Scalar: 32
-          - Scalar: 59
-          - Scalar: 32
-          - Scalar: 97
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 12
-            path: ""
-            content: "\"{ ] [ ; a\""
-  - Value:
-      String:
-        - - Scalar: 4090
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 10
-            path: ""
-            content: "\"\\u{FFA}\""
-  - Value:
-      String:
-        - - Scalar: 719610
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 12
-            path: ""
-            content: "\"\\u{afafa}\""
-  - Value:
-      String:
-        - - Scalar: 44975
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 11
-            path: ""
-            content: "\"\\u{afaf}\""
-  - Value:
-      String:
-        - - Scalar: 2810
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 10
-            path: ""
-            content: "\"\\u{afa}\""
-  - Value:
-      String:
-        - - Scalar: 175
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 9
-            path: ""
-            content: "\"\\u{af}\""
-  - Value:
-      String:
-        - - Scalar: 10
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 8
-            path: ""
-            content: "\"\\u{a}\""
-  - Value:
-      String:
-        - - Scalar: 10
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 7
-            path: ""
-            content: "\"\\x0A\""
-  - Value:
-      String:
-        - - Scalar: 127
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 7
-            path: ""
-            content: "\"\\x7F\""
-  - Value:
-      String:
-        - - Scalar: 97
-          - Scalar: 97
-          - Scalar: 32
-          - Scalar: 92
-          - Scalar: 32
-          - Scalar: 34
-          - Scalar: 32
-          - Scalar: 39
-          - Scalar: 32
-          - Scalar: 10
-          - Scalar: 32
-          - Scalar: 97
-          - Scalar: 97
-          - Scalar: 32
-          - Scalar: 9
-          - Scalar: 32
-          - Scalar: 13
-          - Scalar: 32
-          - Scalar: 32
-          - Scalar: 0
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 30
-            path: ""
-            content: "\"aa \\\\ \\\" \\' \\n aa \\t \\r  \\0\""
-  - Value:
-      String:
-        - - Scalar: 116
-          - Scalar: 101
-          - Scalar: 115
-          - Scalar: 116
-          - Scalar: 32
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 15
-            path: ""
-            content: "\"test 😒€\""
-  - Value:
-      String:
-        - []
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 15
-            path: ""
-            content: "\"😭😂😘\""
-  - Value:
-      String:
-        - []
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 10
-            path: ""
-            content: "\"✋🏿\""
-  - Value:
-      String:
-        - []
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 7
-            path: ""
-            content: "\"🦀\""
-  - Value:
-      String:
-        - []
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 6
-            path: ""
-            content: "\"￿\""
-  - Value:
-      String:
-        - []
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 12
-            path: ""
-            content: "\"<22><><EFBFBD>\""
-  - Value:
-      String:
-        - - Scalar: 65288
-          - Scalar: 65299
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 17
-            path: ""
-            content: "\"（>３<）三\""
-  - Value:
-      String:
-        - - Scalar: 98
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 4
-            path: ""
-            content: "\"b\" // TODO reenabe once #1682 is closed \"(⑅∫°ਊ°)∫\""
-  - Value:
-      String:
-        - - Scalar: 98
-        - span:
-            line_start: 1
-            line_stop: 1
-            col_start: 1
-            col_stop: 4
-            path: ""
-            content: "\"b\" // TODO reenabe once #1682 is closed \"🦀°1\""

tests/parser/expression/literal/string.leo

@@ -1,5 +1,5 @@
 /*
-namespace: ParseExpression
+namespace: Token
 expectation: Pass
 */
 
@@ -32,5 +32,11 @@ expectation: Pass
 "<22><><EFBFBD>"
 
 "（>３<）三"
+"ヽ༼ ಠ益ಠ ༽ﾉ"
+"(╯°□°）╯︵ ┻━┻"
+"┬─┬ ノ( ゜-゜ノ)"
+"( ͡° ͜ʖ ͡°)"
+"b" // TODO reenabe once #1682 is closed "ᕙ(▀̿ĺ̯▀̿ ̿)ᕗ"
+"♥╣[-_-]╠♥"
 "b" // TODO reenabe once #1682 is closed "(⑅∫°ਊ°)∫"
 "b" // TODO reenabe once #1682 is closed "🦀°1"

tests/test-framework/src/error.rs

@@ -27,19 +27,28 @@ pub struct TestFailure {
 
 #[derive(Debug)]
 pub enum TestError {
+    Panicked {
+        test: String,
+        index: usize,
+        error: String,
+    },
     UnexpectedOutput {
+        test: String,
         index: usize,
         expected: Value,
         output: Value,
     },
     PassedAndShouldntHave {
+        test: String,
         index: usize,
     },
     FailedAndShouldntHave {
+        test: String,
         index: usize,
         error: String,
     },
     UnexpectedError {
+        test: String,
         index: usize,
         expected: String,
         output: String,
@@ -50,30 +59,64 @@ pub enum TestError {
 
 impl fmt::Display for TestError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let format_test = |test: &str| -> String {
+            if test.len() > 50 {
+                String::new()
+            } else {
+                format!("\n\n{}\n\n", test)
+            }
+        };
         match self {
+            TestError::Panicked { test, index, error } => {
+                write!(
+                    f,
+                    "test #{}: {}encountered a rust panic:\n{}",
+                    index + 1,
+                    format_test(test),
+                    error
+                )
+            }
             TestError::UnexpectedOutput {
+                test,
                 index,
                 expected,
                 output,
             } => {
                 write!(
                     f,
-                    "test #{} expected\n{}\ngot\n{}",
+                    "test #{}: {}expected\n{}\ngot\n{}",
                     index + 1,
+                    format_test(test),
                     serde_yaml::to_string(&expected).expect("serialization failed"),
                     serde_yaml::to_string(&output).expect("serialization failed")
                 )
             }
-            TestError::PassedAndShouldntHave { index } => write!(f, "test #{} passed and shouldn't have", index + 1),
+            TestError::PassedAndShouldntHave { test, index } => {
-            TestError::FailedAndShouldntHave { index, error } => {
+                write!(f, "test #{}: {}passed and shouldn't have", index + 1, format_test(test))
-                write!(f, "test #{} failed and shouldn't have:\n{}", index + 1, error)
+            }
+            TestError::FailedAndShouldntHave { test, index, error } => {
+                write!(
+                    f,
+                    "test #{}: {}failed and shouldn't have:\n{}",
+                    index + 1,
+                    format_test(test),
+                    error
+                )
             }
             TestError::UnexpectedError {
+                test,
                 expected,
                 output,
                 index,
             } => {
-                write!(f, "test #{} expected error\n{}\ngot\n{}", index + 1, expected, output)
+                write!(
+                    f,
+                    "test #{}: {}expected error\n{}\ngot\n{}",
+                    index + 1,
+                    format_test(test),
+                    expected,
+                    output
+                )
             }
             TestError::MismatchedTestExpectationLength => write!(f, "invalid number of test expectations"),
             TestError::MissingTestConfig => write!(f, "missing test config"),
@@ -82,18 +125,25 @@ impl fmt::Display for TestError {
 }
 
 pub fn emit_errors(
-    output: Result<&Value, &str>,
+    test: &str,
+    output: &Result<Result<Value, String>, String>,
     mode: &TestExpectationMode,
     expected_output: Option<Value>,
     test_index: usize,
 ) -> Option<TestError> {
     match (output, mode) {
-        (Ok(output), TestExpectationMode::Pass) => {
+        (Err(e), _) => Some(TestError::Panicked {
+            test: test.to_string(),
+            index: test_index,
+            error: e.to_string(),
+        }),
+        (Ok(Ok(output)), TestExpectationMode::Pass) => {
             // passed and should have
             if let Some(expected_output) = expected_output.as_ref() {
                 if output != expected_output {
                     // invalid output
                     return Some(TestError::UnexpectedOutput {
+                        test: test.to_string(),
                         index: test_index,
                         expected: expected_output.clone(),
                         output: output.clone(),
@@ -102,18 +152,23 @@ pub fn emit_errors(
             }
             None
         }
-        (Ok(_tokens), TestExpectationMode::Fail) => Some(TestError::PassedAndShouldntHave { index: test_index }),
+        (Ok(Ok(_tokens)), TestExpectationMode::Fail) => Some(TestError::PassedAndShouldntHave {
-        (Err(err), TestExpectationMode::Pass) => Some(TestError::FailedAndShouldntHave {
+            test: test.to_string(),
+            index: test_index,
+        }),
+        (Ok(Err(err)), TestExpectationMode::Pass) => Some(TestError::FailedAndShouldntHave {
+            test: test.to_string(),
             error: err.to_string(),
             index: test_index,
         }),
-        (Err(err), TestExpectationMode::Fail) => {
+        (Ok(Err(err)), TestExpectationMode::Fail) => {
             let expected_output: Option<String> =
                 expected_output.map(|x| serde_yaml::from_value(x).expect("test expectation deserialize failed"));
             if let Some(expected_output) = expected_output.as_deref() {
                 if err != expected_output {
                     // invalid output
                     return Some(TestError::UnexpectedError {
+                        test: test.to_string(),
                         expected: expected_output.to_string(),
                         output: err.to_string(),
                         index: test_index,

tests/test-framework/src/runner.rs

@@ -16,8 +16,12 @@
 
 use serde_yaml::Value;
 use std::{
+    any::Any,
     collections::BTreeMap,
+    panic::{self, RefUnwindSafe, UnwindSafe},
     path::{Path, PathBuf},
+    sync::{Arc, Mutex},
+    thread,
 };
 
 use crate::{error::*, fetch::find_tests, output::TestExpectation, test::*};
@@ -36,7 +40,7 @@ pub struct Test {
     pub config: BTreeMap<String, Value>,
 }
 
-pub trait Namespace {
+pub trait Namespace: UnwindSafe + RefUnwindSafe {
     fn parse_type(&self) -> ParseType;
 
     fn run_test(&self, test: Test) -> Result<Value, String>;
@@ -46,6 +50,36 @@ pub trait Runner {
     fn resolve_namespace(&self, name: &str) -> Option<Box<dyn Namespace>>;
 }
 
+fn set_hook() -> Arc<Mutex<Option<String>>> {
+    let panic_buf = Arc::new(Mutex::new(None));
+    let thread_id = thread::current().id();
+    panic::set_hook({
+        let panic_buf = panic_buf.clone();
+        Box::new(move |e| {
+            if thread::current().id() == thread_id {
+                *panic_buf.lock().unwrap() = Some(e.to_string());
+            } else {
+                println!("{}", e)
+            }
+        })
+    });
+    panic_buf
+}
+
+fn take_hook(
+    output: Result<Result<Value, String>, Box<dyn Any + Send>>,
+    panic_buf: Arc<Mutex<Option<String>>>,
+) -> Result<Result<Value, String>, String> {
+    output.map_err(|_| {
+        panic_buf
+            .lock()
+            .unwrap()
+            .take()
+            .expect("failed to get panic message")
+            .clone()
+    })
+}
+
 pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
     std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files
     std::env::set_var("LEO_TESTFRAMEWORK", "true");
@@ -149,24 +183,24 @@ pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
         for (i, test) in tests.into_iter().enumerate() {
             let expected_output = expected_output.as_mut().and_then(|x| x.next()).cloned();
             println!("running test {} @ '{}'", test_name, path.to_str().unwrap());
-            let output = namespace.run_test(Test {
+            let panic_buf = set_hook();
+            let leo_output = panic::catch_unwind(|| {
+                namespace.run_test(Test {
                     name: test_name.clone(),
                     content: test.clone(),
                     path: path.into(),
                     config: config.extra.clone(),
+                })
             });
-            if let Some(error) = emit_errors(
+            let output = take_hook(leo_output, panic_buf);
-                output.as_ref().map_err(|x| &**x),
+            if let Some(error) = emit_errors(&test, &output, &config.expectation, expected_output, i) {
-                &config.expectation,
-                expected_output,
-                i,
-            ) {
                 fail_tests += 1;
                 errors.push(error);
             } else {
                 pass_tests += 1;
                 new_outputs.push(
                     output
+                        .unwrap()
                         .as_ref()
                         .map(|x| serde_yaml::to_value(x).expect("serialization failed"))
                         .unwrap_or_else(|e| Value::String(e.clone())),