Merge pull request #1865 from AleoHQ/feature/compiler-benchmarking

[Feature] compiler benchmarking

Cargo.lock

@@ -381,16 +381,16 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "3.2.1"
+version = "3.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a836566fa5f52f7ddf909a8a2f9029b9f78ca584cd95cf7e87f8073110f4c5c9"
+checksum = "d53da17d37dba964b9b3ecb5c5a1f193a2762c700e6829201e645b9381c99dc7"
 dependencies = [
  "atty",
  "bitflags",
  "clap_derive",
  "clap_lex",
  "indexmap",
- "lazy_static",
+ "once_cell",
  "strsim",
  "termcolor",
  "textwrap 0.15.0",
@@ -398,9 +398,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "3.2.1"
+version = "3.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "986fd75d1dfd2c34eb8c9275ae38ad87ea9478c9b79e87f1801f7d866dfb1e37"
+checksum = "c11d40217d16aee8508cc8e5fde8b4ff24639758608e5374e731b53f85749fb9"
 dependencies = [
  "heck",
  "proc-macro-error",
@@ -857,13 +857,13 @@ dependencies = [
 
 [[package]]
 name = "getrandom"
-version = "0.2.6"
+version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
+checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6"
 dependencies = [
  "cfg-if",
  "libc",
- "wasi 0.10.2+wasi-snapshot-preview1",
+ "wasi",
 ]
 
 [[package]]
@@ -1094,9 +1094,9 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.57"
+version = "0.3.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397"
+checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1173,7 +1173,7 @@ dependencies = [
  "ansi_term",
  "assert_cmd",
  "backtrace",
- "clap 3.2.1",
+ "clap 3.2.5",
  "color-backtrace",
  "colored",
  "console",
@@ -1216,8 +1216,7 @@ dependencies = [
 name = "leo-parser"
 version = "1.5.3"
 dependencies = [
- "clap 3.2.1",
+ "clap 3.2.5",
- "criterion",
  "indexmap",
  "lazy_static",
  "leo-ast",
@@ -1258,12 +1257,17 @@ dependencies = [
 name = "leo-test-framework"
 version = "1.5.3"
 dependencies = [
- "clap 3.2.1",
+ "backtrace",
+ "clap 3.2.5",
+ "criterion",
+ "leo-compiler",
  "leo-errors",
+ "leo-span",
  "regex",
  "serde",
  "serde_json",
  "serde_yaml",
+ "walkdir",
 ]
 
 [[package]]
@@ -1369,7 +1373,7 @@ checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799"
 dependencies = [
  "libc",
  "log",
- "wasi 0.11.0+wasi-snapshot-preview1",
+ "wasi",
  "windows-sys",
 ]
 
@@ -2015,9 +2019,9 @@ dependencies = [
 
 [[package]]
 name = "semver"
-version = "1.0.9"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd"
+checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c"
 
 [[package]]
 name = "serde"
@@ -2661,9 +2665,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c"
 
 [[package]]
 name = "unicode-normalization"
@@ -2746,12 +2750,6 @@ dependencies = [
  "try-lock",
 ]
 
-[[package]]
-name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
-
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -2760,9 +2758,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad"
+checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994"
 dependencies = [
  "cfg-if",
  "wasm-bindgen-macro",
@@ -2770,9 +2768,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4"
+checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a"
 dependencies = [
  "bumpalo",
  "lazy_static",
@@ -2785,9 +2783,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.30"
+version = "0.4.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2"
+checksum = "de9a9cec1733468a8c657e57fa2413d2ae2c0129b95e87c5b72b8ace4d13f31f"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -2797,9 +2795,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5"
+checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa"
 dependencies = [
  "quote 1.0.18",
  "wasm-bindgen-macro-support",
@@ -2807,9 +2805,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b"
+checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048"
 dependencies = [
  "proc-macro2 1.0.39",
  "quote 1.0.18",
@@ -2820,15 +2818,15 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.80"
+version = "0.2.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744"
+checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be"
 
 [[package]]
 name = "web-sys"
-version = "0.3.57"
+version = "0.3.58"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283"
+checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90"
 dependencies = [
  "js-sys",
  "wasm-bindgen",

compiler/ast/src/input/input_ast.rs

@@ -46,6 +46,15 @@ impl InputAst {
         Ok(serde_json::to_value(&self).map_err(|e| AstError::failed_to_convert_ast_to_json_value(&e))?)
     }
 
+    /// Serializes the input into a JSON file.
+    pub fn to_json_file(&self, mut path: std::path::PathBuf, file_name: &str) -> Result<()> {
+        path.push(file_name);
+        let file = std::fs::File::create(&path).map_err(|e| AstError::failed_to_create_ast_json_file(&path, &e))?;
+        let writer = std::io::BufWriter::new(file);
+        Ok(serde_json::to_writer_pretty(writer, &self)
+            .map_err(|e| AstError::failed_to_write_ast_to_json_file(&path, &e))?)
+    }
+
     /// Serializes the `Input` into a JSON value and removes keys from object mappings before writing to a file.
     pub fn to_json_file_without_keys(
         &self,

compiler/compiler/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "leo-compiler"
 version = "1.5.3"
-authors = [ "The Aleo Team <hello@aleo.org>" ]
+authors = ["The Aleo Team <hello@aleo.org>"]
 description = "Compiler of the Leo programming language"
 homepage = "https://aleo.org"
 repository = "https://github.com/AleoHQ/leo"
@@ -10,10 +10,10 @@ keywords = [
   "cryptography",
   "leo",
   "programming-language",
-  "zero-knowledge"
+  "zero-knowledge",
 ]
-categories = [ "cryptography::cryptocurrencies", "web-programming" ]
+categories = ["cryptography::cryptocurrencies", "web-programming"]
-include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ]
+include = ["Cargo.toml", "src", "README.md", "LICENSE.md"]
 license = "GPL-3.0"
 edition = "2021"
 rust-version = "1.56.1"
@@ -52,7 +52,6 @@ features = ["derive"]
 [dev-dependencies.serde_yaml]
 version = "0.8.24"
 
-
 [features]
-default = [ ]
+default = []
-ci_skip = [ "leo-ast/ci_skip" ]
+ci_skip = ["leo-ast/ci_skip"]

compiler/compiler/src/compiler.rs

@@ -0,0 +1,176 @@
+// Copyright (C) 2019-2021 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
+
+//! The compiler for Leo programs.
+//!
+//! The [`Compiler`] type compiles Leo programs into R1CS circuits.
+use leo_ast::Program;
+pub use leo_ast::{Ast, InputAst};
+use leo_errors::emitter::Handler;
+use leo_errors::{CompilerError, Result};
+pub use leo_passes::SymbolTable;
+use leo_passes::*;
+use leo_span::source_map::FileName;
+use leo_span::symbol::with_session_globals;
+
+use sha2::{Digest, Sha256};
+use std::fs;
+use std::path::PathBuf;
+
+use crate::OutputOptions;
+
+/// The primary entry point of the Leo compiler.
+#[derive(Clone)]
+pub struct Compiler<'a> {
+    /// The handler is used for error and warning emissions.
+    handler: &'a Handler,
+    /// The path to the main leo file.
+    main_file_path: PathBuf,
+    /// The path to where the compiler outputs all generated files.
+    output_directory: PathBuf,
+    /// The AST for the program.
+    pub ast: Ast,
+    /// The input ast for the program if it exists.
+    pub input_ast: Option<InputAst>,
+    /// Compiler options on some optional output files.
+    output_options: OutputOptions,
+}
+
+impl<'a> Compiler<'a> {
+    ///
+    /// Returns a new Leo compiler.
+    ///
+    pub fn new(
+        handler: &'a Handler,
+        main_file_path: PathBuf,
+        output_directory: PathBuf,
+        output_options: Option<OutputOptions>,
+    ) -> Self {
+        Self {
+            handler,
+            main_file_path,
+            output_directory,
+            ast: Ast::new(Program::new("Initial".to_string())),
+            input_ast: None,
+            output_options: output_options.unwrap_or_default(),
+        }
+    }
+
+    ///
+    /// Returns a SHA256 checksum of the program file.
+    ///
+    pub fn checksum(&self) -> Result<String> {
+        // Read in the main file as string
+        let unparsed_file = fs::read_to_string(&self.main_file_path)
+            .map_err(|e| CompilerError::file_read_error(self.main_file_path.clone(), e))?;
+
+        // Hash the file contents
+        let mut hasher = Sha256::new();
+        hasher.update(unparsed_file.as_bytes());
+        let hash = hasher.finalize();
+
+        Ok(format!("{:x}", hash))
+    }
+
+    // Parses and stores a program file content from a string, constructs a syntax tree, and generates a program.
+    pub fn parse_program_from_string(&mut self, program_string: &str, name: FileName) -> Result<()> {
+        // Register the source (`program_string`) in the source map.
+        let prg_sf = with_session_globals(|s| s.source_map.new_source(program_string, name));
+
+        // Use the parser to construct the abstract syntax tree (ast).
+        let ast: leo_ast::Ast = leo_parser::parse_ast(self.handler, &prg_sf.src, prg_sf.start_pos)?;
+
+        if self.output_options.ast_initial {
+            // Write the AST snapshot post parsing.
+            if self.output_options.spans_enabled {
+                ast.to_json_file(self.output_directory.clone(), "initial_ast.json")?;
+            } else {
+                ast.to_json_file_without_keys(self.output_directory.clone(), "initial_ast.json", &["span"])?;
+            }
+        }
+
+        self.ast = ast;
+
+        Ok(())
+    }
+
+    /// Parses and stores the main program file, constructs a syntax tree, and generates a program.
+    pub fn parse_program(&mut self) -> Result<()> {
+        // Load the program file.
+        let program_string = fs::read_to_string(&self.main_file_path)
+            .map_err(|e| CompilerError::file_read_error(&self.main_file_path, e))?;
+
+        self.parse_program_from_string(&program_string, FileName::Real(self.main_file_path.clone()))
+    }
+
+    /// Parses and stores the input file, constructs a syntax tree, and generates a program input.
+    pub fn parse_input(&mut self, input_file_path: PathBuf) -> Result<()> {
+        if input_file_path.exists() {
+            // Load the input file into the source map.
+            let input_sf = with_session_globals(|s| s.source_map.load_file(&input_file_path))
+                .map_err(|e| CompilerError::file_read_error(&input_file_path, e))?;
+
+            // Parse and serialize it.
+            let input_ast = leo_parser::parse_input(self.handler, &input_sf.src, input_sf.start_pos)?;
+            if self.output_options.ast_initial {
+                // Write the input AST snapshot post parsing.
+                if self.output_options.spans_enabled {
+                    input_ast.to_json_file(self.output_directory.clone(), "initial_input_ast.json")?;
+                } else {
+                    input_ast.to_json_file_without_keys(
+                        self.output_directory.clone(),
+                        "initial_input_ast.json",
+                        &["span"],
+                    )?;
+                }
+            }
+
+            self.input_ast = Some(input_ast);
+        }
+        Ok(())
+    }
+
+    ///
+    /// Runs the symbol table pass.
+    ///
+    pub fn symbol_table_pass(&self) -> Result<SymbolTable<'_>> {
+        CreateSymbolTable::do_pass((&self.ast, self.handler))
+    }
+
+    ///
+    /// Runs the type checker pass.
+    ///
+    pub fn type_checker_pass(&'a self, symbol_table: &mut SymbolTable<'_>) -> Result<()> {
+        TypeChecker::do_pass((&self.ast, &mut symbol_table.clone(), self.handler))
+    }
+
+    ///
+    /// Runs the compiler stages.
+    ///
+    pub fn compiler_stages(&self) -> Result<SymbolTable<'_>> {
+        let mut st = self.symbol_table_pass()?;
+        self.type_checker_pass(&mut st)?;
+        Ok(st)
+    }
+
+    ///
+    /// Returns a compiled Leo program.
+    ///
+    pub fn compile(&mut self) -> Result<SymbolTable<'_>> {
+        self.parse_program()?;
+        self.compiler_stages()
+    }
+}

compiler/compiler/src/lib.rs

@@ -14,125 +14,15 @@
 // You should have received a copy of the GNU General Public License
 // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
-//! The compiler for Leo programs.
-//!
-//! The [`Compiler`] type compiles Leo programs into R1CS circuits.
-
 #![allow(clippy::module_inception)]
 #![allow(clippy::upper_case_acronyms)]
 #![doc = include_str!("../README.md")]
 
+mod compiler;
+pub use compiler::*;
+
+mod options;
+pub use options::*;
+
 #[cfg(test)]
 mod test;
-
-use leo_ast::Program;
-pub use leo_ast::{Ast, InputAst};
-use leo_errors::emitter::Handler;
-use leo_errors::{CompilerError, Result};
-pub use leo_passes::SymbolTable;
-use leo_passes::*;
-use leo_span::source_map::FileName;
-use leo_span::symbol::with_session_globals;
-
-use sha2::{Digest, Sha256};
-use std::fs;
-use std::path::PathBuf;
-
-#[derive(Clone)]
-/// The primary entry point of the Leo compiler.
-pub struct Compiler<'a> {
-    handler: &'a Handler,
-    main_file_path: PathBuf,
-    output_directory: PathBuf,
-    pub ast: Ast,
-    pub input_ast: Option<InputAst>,
-}
-
-impl<'a> Compiler<'a> {
-    ///
-    /// Returns a new Leo compiler.
-    ///
-    pub fn new(handler: &'a Handler, main_file_path: PathBuf, output_directory: PathBuf) -> Self {
-        Self {
-            handler,
-            main_file_path,
-            output_directory,
-            ast: Ast::new(Program::new("Initial".to_string())),
-            input_ast: None,
-        }
-    }
-
-    ///
-    /// Returns a SHA256 checksum of the program file.
-    ///
-    pub fn checksum(&self) -> Result<String> {
-        // Read in the main file as string
-        let unparsed_file = fs::read_to_string(&self.main_file_path)
-            .map_err(|e| CompilerError::file_read_error(self.main_file_path.clone(), e))?;
-
-        // Hash the file contents
-        let mut hasher = Sha256::new();
-        hasher.update(unparsed_file.as_bytes());
-        let hash = hasher.finalize();
-
-        Ok(format!("{:x}", hash))
-    }
-
-    // Parses and stores a program file content from a string, constructs a syntax tree, and generates a program.
-    pub fn parse_program_from_string(&mut self, program_string: &str, name: FileName) -> Result<()> {
-        // Register the source (`program_string`) in the source map.
-        let prg_sf = with_session_globals(|s| s.source_map.new_source(program_string, name));
-
-        // Use the parser to construct the abstract syntax tree (ast).
-        let ast: leo_ast::Ast = leo_parser::parse_ast(self.handler, &prg_sf.src, prg_sf.start_pos)?;
-        // Write the AST snapshot post parsing.
-        ast.to_json_file_without_keys(self.output_directory.clone(), "initial_ast.json", &["span"])?;
-
-        self.ast = ast;
-
-        Ok(())
-    }
-
-    /// Parses and stores the main program file, constructs a syntax tree, and generates a program.
-    pub fn parse_program(&mut self) -> Result<()> {
-        // Load the program file.
-        let program_string = fs::read_to_string(&self.main_file_path)
-            .map_err(|e| CompilerError::file_read_error(&self.main_file_path, e))?;
-
-        self.parse_program_from_string(&program_string, FileName::Real(self.main_file_path.clone()))
-    }
-
-    /// Parses and stores the input file, constructs a syntax tree, and generates a program input.
-    pub fn parse_input(&mut self, input_file_path: PathBuf) -> Result<()> {
-        if input_file_path.exists() {
-            // Load the input file into the source map.
-            let input_sf = with_session_globals(|s| s.source_map.load_file(&input_file_path))
-                .map_err(|e| CompilerError::file_read_error(&input_file_path, e))?;
-
-            // Parse and serialize it.
-            let input_ast = leo_parser::parse_input(self.handler, &input_sf.src, input_sf.start_pos)?;
-            input_ast.to_json_file_without_keys(self.output_directory.clone(), "initial_input_ast.json", &["span"])?;
-
-            self.input_ast = Some(input_ast);
-        }
-        Ok(())
-    }
-
-    ///
-    /// Runs the compiler stages.
-    ///
-    fn compiler_stages(&self) -> Result<SymbolTable<'_>> {
-        let symbol_table = CreateSymbolTable::do_pass((&self.ast, self.handler))?;
-
-        TypeChecker::do_pass((&self.ast, &mut symbol_table.clone(), self.handler))?;
-
-        Ok(symbol_table)
-    }
-
-    ///
-    /// Returns a compiled Leo program.
-    ///
-    pub fn compile(&self) -> Result<SymbolTable<'_>> {
-        self.compiler_stages()
-    }
-}

compiler/compiler/src/options.rs

@@ -0,0 +1,25 @@
+// Copyright (C) 2019-2021 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
+
+#[derive(Clone, Default)]
+pub struct OutputOptions {
+    /// Whether spans are enabled in the output ASTs.
+    pub spans_enabled: bool,
+    /// If enabled writes the AST after parsing.
+    pub ast_initial: bool,
+    /// If enabled writes the input AST after parsing.
+    pub input_ast_initial: bool,
+}

compiler/compiler/src/test.rs

@@ -21,7 +21,7 @@ use std::{
     rc::Rc,
 };
 
-use crate::Compiler;
+use crate::{Compiler, OutputOptions};
 
 use leo_errors::{
     emitter::{Buffer, Emitter, Handler},
@@ -40,7 +40,16 @@ fn new_compiler(handler: &Handler, main_file_path: PathBuf) -> Compiler<'_> {
     let output_dir = PathBuf::from("/tmp/output/");
     fs::create_dir_all(output_dir.clone()).unwrap();
 
-    Compiler::new(handler, main_file_path, output_dir)
+    Compiler::new(
+        handler,
+        main_file_path,
+        output_dir,
+        Some(OutputOptions {
+            spans_enabled: false,
+            input_ast_initial: true,
+            ast_initial: true,
+        }),
+    )
 }
 
 fn parse_program<'a>(
@@ -123,8 +132,10 @@ fn collect_all_inputs(test: &Test) -> Result<Vec<PathBuf>, String> {
     Ok(list)
 }
 
-fn compile_and_process<'a>(parsed: &'a mut Compiler<'a>) -> Result<SymbolTable<'a>, LeoError> {
+fn compile_and_process<'a>(parsed: &'a mut Compiler<'a>) -> Result<SymbolTable<'_>, LeoError> {
-    parsed.compiler_stages()
+    let mut st = parsed.symbol_table_pass()?;
+    parsed.type_checker_pass(&mut st)?;
+    Ok(st)
 }
 
 // Errors used in this module.

compiler/parser/Cargo.toml

@@ -18,11 +18,6 @@ license = "GPL-3.0"
 edition = "2021"
 rust-version = "1.56"
 
-[[bench]]
-name = "leo_ast"
-path = "benches/leo_ast.rs"
-harness = false
-
 [dependencies.leo-ast]
 path = "../ast"
 version = "1.5.3"
@@ -59,9 +54,6 @@ version = "0.1"
 path = "../../tests/test-framework"
 version = "1.4.0"
 
-[dev-dependencies.criterion]
-version = "0.3"
-
 [dev-dependencies.serde_json]
 version = "1.0"
 features = [ "preserve_order" ]

compiler/parser/benches/leo_ast.rs

@@ -1,96 +0,0 @@
-// Copyright (C) 2019-2022 Aleo Systems Inc.
-// This file is part of the Leo library.
-
-// The Leo library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// The Leo library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
-
-use leo_ast::Ast;
-use leo_errors::emitter::Handler;
-use leo_span::{source_map::FileName, symbol::create_session_if_not_set_then};
-
-use criterion::{criterion_group, criterion_main, Criterion};
-use std::time::Duration;
-
-fn parse_ast(path: &str, input: &str) -> Ast {
-    create_session_if_not_set_then(|s| {
-        let sf = s.source_map.new_source(input, FileName::Custom(path.into()));
-        leo_parser::parse_ast(&Handler::default(), &sf.src, sf.start_pos).expect("failed to parse benchmark")
-    })
-}
-
-macro_rules! bench {
-    ($func_name:ident, $file_name:expr) => {
-        fn $func_name(c: &mut Criterion) {
-            let ast = parse_ast(
-                concat!("./", $file_name, ".leo"),
-                include_str!(concat!("./", $file_name, ".leo"),),
-            );
-            // TODO(Centril): This benchmark seems like it actually does nothing
-            // but take a reference to `&ast`, which should be optimized out?
-            c.bench_function(concat!("Ast::", $file_name), |b| b.iter(|| &ast));
-        }
-    };
-}
-
-bench!(bench_big_if_else, "big_if_else");
-// TODO have to figure out why this causes `thread 'main' has overflowed it's stack'
-// bench!(bench_big_ternary, "big_ternary");
-bench!(bench_big_circuit, "big_circuit");
-bench!(bench_long_expr, "long_expr");
-bench!(bench_long_array, "long_array");
-bench!(bench_many_foos, "many_foos");
-bench!(bench_many_assigns, "many_assigns");
-bench!(bench_small_1, "small_1");
-bench!(bench_small_2, "small_2");
-bench!(bench_small_3, "small_3");
-bench!(bench_small_4, "small_4");
-bench!(bench_small_5, "small_5");
-bench!(bench_medium_1, "medium_1");
-bench!(bench_medium_2, "medium_2");
-bench!(bench_medium_3, "medium_3");
-bench!(bench_medium_4, "medium_4");
-bench!(bench_medium_5, "medium_5");
-bench!(bench_large_1, "large_1");
-bench!(bench_large_2, "large_2");
-bench!(bench_large_3, "large_3");
-bench!(bench_large_4, "large_4");
-bench!(bench_large_5, "large_5");
-bench!(bench_massive, "massive");
-
-criterion_group!(
-    name = benches;
-    config = Criterion::default().sample_size(200).measurement_time(Duration::from_secs(10)).nresamples(200_000);
-    targets = bench_big_circuit,
-    bench_long_expr,
-    bench_big_if_else,
-    bench_long_array,
-    bench_many_assigns,
-    bench_many_foos,
-    bench_small_1,
-    bench_small_2,
-    bench_small_3,
-    bench_small_4,
-    bench_small_5,
-    bench_medium_1,
-    bench_medium_2,
-    bench_medium_3,
-    bench_medium_4,
-    bench_medium_5,
-    bench_large_1,
-    bench_large_2,
-    bench_large_3,
-    bench_large_4,
-    bench_large_5,
-    bench_massive
-);
-criterion_main!(benches);

compiler/parser/benches/long_expr.leo

@@ -1,9 +0,0 @@
-function main() {
-    const a = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1;
-    const b = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1;
-    const c = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1;
-    const d = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1;
-    const e = 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1;
-
-    return a + b + c + d + e;
-}

compiler/passes/src/pass.rs

@@ -15,7 +15,7 @@
 // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
 /// A pass consuming a `Program` and possibly returning an `Ast`.
-pub trait Pass<'a> {
+pub trait Pass {
     type Input;
     type Output;
 

compiler/passes/src/symbol_table/mod.rs

@@ -34,7 +34,7 @@ use crate::Pass;
 use leo_ast::{Ast, ProgramVisitorDirector, VisitorDirector};
 use leo_errors::{emitter::Handler, Result};
 
-impl<'a> Pass<'a> for CreateSymbolTable<'a> {
+impl<'a> Pass for CreateSymbolTable<'a> {
     type Input = (&'a Ast, &'a Handler);
     type Output = Result<SymbolTable<'a>>;
 

compiler/passes/src/type_checker/mod.rs

@@ -34,7 +34,7 @@ use crate::{Pass, SymbolTable};
 use leo_ast::{Ast, ProgramVisitorDirector};
 use leo_errors::{emitter::Handler, Result};
 
-impl<'a> Pass<'a> for TypeChecker<'a> {
+impl<'a> Pass for TypeChecker<'a> {
     type Input = (&'a Ast, &'a mut SymbolTable<'a>, &'a Handler);
     type Output = Result<()>;
 

leo/commands/build.rs

@@ -15,7 +15,7 @@
 // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
 use crate::{commands::Command, context::Context};
-use leo_compiler::{Ast, Compiler, InputAst};
+use leo_compiler::{Ast, Compiler, InputAst, OutputOptions};
 use leo_errors::{CliError, Result};
 use leo_package::{
     inputs::InputFile,
@@ -32,68 +32,35 @@ use tracing::span::Span;
 /// require Build command output as their input.
 #[derive(StructOpt, Clone, Debug, Default)]
 pub struct BuildOptions {
-    #[structopt(long, help = "Disable constant folding compiler optimization")]
-    pub disable_constant_folding: bool,
-    #[structopt(long, help = "Disable dead code elimination compiler optimization")]
-    pub disable_code_elimination: bool,
-    #[structopt(long, help = "Disable all compiler optimizations")]
-    pub disable_all_optimizations: bool,
     #[structopt(long, help = "Enable spans in AST snapshots.")]
     pub enable_spans: bool,
     #[structopt(long, help = "Writes all AST snapshots for the different compiler phases.")]
     pub enable_all_ast_snapshots: bool,
+    #[structopt(long, help = "Writes Input AST snapshot of the initial parse.")]
+    pub enable_initial_input_ast_snapshot: bool,
     #[structopt(long, help = "Writes AST snapshot of the initial parse.")]
     pub enable_initial_ast_snapshot: bool,
-    #[structopt(long, help = "Writes AST snapshot after the import resolution phase.")]
-    pub enable_imports_resolved_ast_snapshot: bool,
-    #[structopt(long, help = "Writes AST snapshot after the canonicalization phase.")]
-    pub enable_canonicalized_ast_snapshot: bool,
-    #[structopt(long, help = "Writes AST snapshot after the type inference phase.")]
-    pub enable_type_inferenced_ast_snapshot: bool,
 }
 
-// impl From<BuildOptions> for CompilerOptions {
+impl From<BuildOptions> for OutputOptions {
-//     fn from(options: BuildOptions) -> Self {
+    fn from(options: BuildOptions) -> Self {
-//         if options.disable_all_optimizations {
+        let mut out_options = Self {
-//             CompilerOptions {
+            spans_enabled: options.enable_spans,
-//                 constant_folding_enabled: false,
+            input_ast_initial: options.enable_initial_input_ast_snapshot,
-//                 dead_code_elimination_enabled: false,
+            ast_initial: options.enable_initial_ast_snapshot,
-//             }
+        };
-//         } else {
+        if options.enable_all_ast_snapshots {
-//             CompilerOptions {
+            out_options.input_ast_initial = true;
-//                 constant_folding_enabled: !options.disable_constant_folding,
+            out_options.ast_initial = true;
-//                 dead_code_elimination_enabled: !options.disable_code_elimination,
+        }
-//             }
-//         }
-//     }
-// }
 
-// impl From<BuildOptions> for AstSnapshotOptions {
+        out_options
-//     fn from(options: BuildOptions) -> Self {
+    }
-//         if options.enable_all_ast_snapshots {
+}
-//             AstSnapshotOptions {
-//                 spans_enabled: options.enable_spans,
-//                 initial: true,
-//                 imports_resolved: true,
-//                 canonicalized: true,
-//                 type_inferenced: true,
-//             }
-//         } else {
-//             AstSnapshotOptions {
-//                 spans_enabled: options.enable_spans,
-//                 initial: options.enable_initial_ast_snapshot,
-//                 imports_resolved: options.enable_imports_resolved_ast_snapshot,
-//                 canonicalized: options.enable_canonicalized_ast_snapshot,
-//                 type_inferenced: options.enable_type_inferenced_ast_snapshot,
-//             }
-//         }
-//     }
-// }
 
 /// Compile and build program command.
 #[derive(StructOpt, Debug)]
 pub struct Build {
-    #[allow(dead_code)]
     #[structopt(flatten)]
     pub(crate) compiler_options: BuildOptions,
 }
@@ -179,8 +146,12 @@ impl Command for Build {
         // Initialize error handler
         let handler = leo_errors::emitter::Handler::default();
 
-        let mut program = Compiler::new(&handler, main_file_path, output_directory);
+        let mut program = Compiler::new(
-        program.parse_program()?;
+            &handler,
+            main_file_path,
+            output_directory,
+            Some(self.compiler_options.into()),
+        );
         program.parse_input(input_path.to_path_buf())?;
 
         // Compute the current program checksum

leo/span/src/symbol.rs

@@ -293,8 +293,8 @@ pub struct SessionGlobals {
     pub source_map: SourceMap,
 }
 
-impl SessionGlobals {
+impl Default for SessionGlobals {
-    fn new() -> Self {
+    fn default() -> Self {
         Self {
             symbol_interner: Interner::prefilled(),
             source_map: SourceMap::default(),
@@ -302,13 +302,13 @@ impl SessionGlobals {
     }
 }
 
-scoped_tls::scoped_thread_local!(static SESSION_GLOBALS: SessionGlobals);
+scoped_tls::scoped_thread_local!(pub static SESSION_GLOBALS: SessionGlobals);
 
 /// Creates the session globals and then runs the closure `f`.
 #[inline]
 pub fn create_session_if_not_set_then<R>(f: impl FnOnce(&SessionGlobals) -> R) -> R {
     if !SESSION_GLOBALS.is_set() {
-        let sg = SessionGlobals::new();
+        let sg = SessionGlobals::default();
         SESSION_GLOBALS.set(&sg, || SESSION_GLOBALS.with(f))
     } else {
         SESSION_GLOBALS.with(f)

tests/compiler/additional_benches/big_circuit.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 function main() {
     const foo = Foo { x0: 0, x1: 1, x2: 2, x3: 3, x4: 4, x5: 5, x6: 6, x7: 7, x8: 8, x9: 9, x10: 10, x11: 11, x12: 12, x13: 13, x14: 14, x15: 15, x16: 16, x17: 17, x18: 18, x19: 19, x20: 20, x21: 21, x22: 22, x23: 23, x24: 24, x25: 25, x26: 26, x27: 27, x28: 28, x29: 29, x30: 30, x31: 31, x32: 32, x33: 33, x34: 34, x35: 35, x36: 36, x37: 37, x38: 38, x39: 39, x40: 40, x41: 41, x42: 42, x43: 43, x44: 44, x45: 45, x46: 46, x47: 47, x48: 48, x49: 49, x50: 50, x51: 51, x52: 52, x53: 53, x54: 54, x55: 55, x56: 56, x57: 57, x58: 58, x59: 59, x60: 60, x61: 61, x62: 62, x63: 63, x64: 64, x65: 65, x66: 66, x67: 67, x68: 68, x69: 69, x70: 70, x71: 71, x72: 72, x73: 73, x74: 74, x75: 75, x76: 76, x77: 77, x78: 78, x79: 79, x80: 80, x81: 81, x82: 82, x83: 83, x84: 84, x85: 85, x86: 86, x87: 87, x88: 88, x89: 89, x90: 90, x91: 91, x92: 92, x93: 93, x94: 94, x95: 95, x96: 96, x97: 97, x98: 98, x99: 99, x100: 100, x101: 101, x102: 102, x103: 103, x104: 104, x105: 105, x106: 106, x107: 107, x108: 108, x109: 109, x110: 110, x111: 111, x112: 112, x113: 113, x114: 114, x115: 115, x116: 116, x117: 117, x118: 118, x119: 119, x120: 120, x121: 121, x122: 122, x123: 123, x124: 124, x125: 125, x126: 126, x127: 127, x128: 128, x129: 129, x130: 130, x131: 131, x132: 132, x133: 133, x134: 134, x135: 135, x136: 136, x137: 137, x138: 138, x139: 139, x140: 140, x141: 141, x142: 142, x143: 143, x144: 144, x145: 145, x146: 146, x147: 147, x148: 148, x149: 149, x150: 150, x151: 151, x152: 152, x153: 153, x154: 154, x155: 155, x156: 156, x157: 157, x158: 158, x159: 159, x160: 160, x161: 161, x162: 162, x163: 163, x164: 164, x165: 165, x166: 166, x167: 167, x168: 168, x169: 169, x170: 170, x171: 171, x172: 172, x173: 173, x174: 174, x175: 175, x176: 176, x177: 177, x178: 178, x179: 179, x180: 180, x181: 181, x182: 182, x183: 183, x184: 184, x185: 185, x186: 186, x187: 187, x188: 188, x189: 189, x190: 190, x191: 191, x192: 192, x193: 193, x194: 194, x195: 195, x196: 196, x197: 197, x198: 198, x199: 199, x200: 200, x201: 201, x202: 202, x203: 203, x204: 204, x205: 205, x206: 206, x207: 207, x208: 208, x209: 209, x210: 210, x211: 211, x212: 212, x213: 213, x214: 214, x215: 215, x216: 216, x217: 217, x218: 218, x219: 219, x220: 220, x221: 221, x222: 222, x223: 223, x224: 224, x225: 225, x226: 226, x227: 227, x228: 228, x229: 229, x230: 230, x231: 231, x232: 232, x233: 233, x234: 234, x235: 235, x236: 236, x237: 237, x238: 238, x239: 239, x240: 240, x241: 241, x242: 242, x243: 243, x244: 244, x245: 245, x246: 246, x247: 247, x248: 248, x249: 249, x250: 250, x251: 251, x252: 252, x253: 253, x254: 254, x255: 255 };
     const bar = Foo { x0: 0, x1: 1, x2: 2, x3: 3, x4: 4, x5: 5, x6: 6, x7: 7, x8: 8, x9: 9, x10: 10, x11: 11, x12: 12, x13: 13, x14: 14, x15: 15, x16: 16, x17: 17, x18: 18, x19: 19, x20: 20, x21: 21, x22: 22, x23: 23, x24: 24, x25: 25, x26: 26, x27: 27, x28: 28, x29: 29, x30: 30, x31: 31, x32: 32, x33: 33, x34: 34, x35: 35, x36: 36, x37: 37, x38: 38, x39: 39, x40: 40, x41: 41, x42: 42, x43: 43, x44: 44, x45: 45, x46: 46, x47: 47, x48: 48, x49: 49, x50: 50, x51: 51, x52: 52, x53: 53, x54: 54, x55: 55, x56: 56, x57: 57, x58: 58, x59: 59, x60: 60, x61: 61, x62: 62, x63: 63, x64: 64, x65: 65, x66: 66, x67: 67, x68: 68, x69: 69, x70: 70, x71: 71, x72: 72, x73: 73, x74: 74, x75: 75, x76: 76, x77: 77, x78: 78, x79: 79, x80: 80, x81: 81, x82: 82, x83: 83, x84: 84, x85: 85, x86: 86, x87: 87, x88: 88, x89: 89, x90: 90, x91: 91, x92: 92, x93: 93, x94: 94, x95: 95, x96: 96, x97: 97, x98: 98, x99: 99, x100: 100, x101: 101, x102: 102, x103: 103, x104: 104, x105: 105, x106: 106, x107: 107, x108: 108, x109: 109, x110: 110, x111: 111, x112: 112, x113: 113, x114: 114, x115: 115, x116: 116, x117: 117, x118: 118, x119: 119, x120: 120, x121: 121, x122: 122, x123: 123, x124: 124, x125: 125, x126: 126, x127: 127, x128: 128, x129: 129, x130: 130, x131: 131, x132: 132, x133: 133, x134: 134, x135: 135, x136: 136, x137: 137, x138: 138, x139: 139, x140: 140, x141: 141, x142: 142, x143: 143, x144: 144, x145: 145, x146: 146, x147: 147, x148: 148, x149: 149, x150: 150, x151: 151, x152: 152, x153: 153, x154: 154, x155: 155, x156: 156, x157: 157, x158: 158, x159: 159, x160: 160, x161: 161, x162: 162, x163: 163, x164: 164, x165: 165, x166: 166, x167: 167, x168: 168, x169: 169, x170: 170, x171: 171, x172: 172, x173: 173, x174: 174, x175: 175, x176: 176, x177: 177, x178: 178, x179: 179, x180: 180, x181: 181, x182: 182, x183: 183, x184: 184, x185: 185, x186: 186, x187: 187, x188: 188, x189: 189, x190: 190, x191: 191, x192: 192, x193: 193, x194: 194, x195: 195, x196: 196, x197: 197, x198: 198, x199: 199, x200: 200, x201: 201, x202: 202, x203: 203, x204: 204, x205: 205, x206: 206, x207: 207, x208: 208, x209: 209, x210: 210, x211: 211, x212: 212, x213: 213, x214: 214, x215: 215, x216: 216, x217: 217, x218: 218, x219: 219, x220: 220, x221: 221, x222: 222, x223: 223, x224: 224, x225: 225, x226: 226, x227: 227, x228: 228, x229: 229, x230: 230, x231: 231, x232: 232, x233: 233, x234: 234, x235: 235, x236: 236, x237: 237, x238: 238, x239: 239, x240: 240, x241: 241, x242: 242, x243: 243, x244: 244, x245: 245, x246: 246, x247: 247, x248: 248, x249: 249, x250: 250, x251: 251, x252: 252, x253: 253, x254: 254, x255: 255 };

tests/compiler/additional_benches/big_if_else.leo

@@ -1,4 +1,9 @@
-function main() {
+/*
+namespace: Bench
+expectation: Skip
+*/
+
+function main() -> u8 {
     const x: u8 = 191;
 
     if x == 0 {

tests/compiler/additional_benches/big_ternary.leo

@@ -1,4 +1,9 @@
-function main() {
+/*
+namespace: Bench
+expectation: Skip
+*/
+
+function main() -> u8 {
     const x: u8 = 255;
 
     return x == 0 ? x : (x == 1 ? x : (x == 2 ? x : (x == 3 ? x : (x == 4 ? x : (x == 5 ? x : (x == 6 ? x : (x == 7 ? x : (x == 8 ? x : (x == 9 ? x : (x == 10 ? x : (x == 11 ? x : (x == 12 ? x : (x == 13 ? x : (x == 14 ? x : (x == 15 ? x : (x == 16 ? x : (x == 17 ? x : (x == 18 ? x : (x == 19 ? x : (x == 20 ? x : (x == 21 ? x : (x == 22 ? x : (x == 23 ? x : (x == 24 ? x : (x == 25 ? x : (x == 26 ? x : (x == 27 ? x : (x == 28 ? x : (x == 29 ? x : (x == 30 ? x : (x == 31 ? x : (x == 32 ? x : (x == 33 ? x : (x == 34 ? x : (x == 35 ? x : (x == 36 ? x : (x == 37 ? x : (x == 38 ? x : (x == 39 ? x : (x == 40 ? x : (x == 41 ? x : (x == 42 ? x : (x == 43 ? x : (x == 44 ? x : (x == 45 ? x : (x == 46 ? x : (x == 47 ? x : (x == 48 ? x : (x == 49 ? x : (x == 50 ? x : (x == 51 ? x : (x == 52 ? x : (x == 53 ? x : (x == 54 ? x : (x == 55 ? x : (x == 56 ? x : (x == 57 ? x : (x == 58 ? x : (x == 59 ? x : (x == 60 ? x : (x == 61 ? x : (x == 62 ? x : (x == 63 ? x : (x == 64 ? x : (x == 65 ? x : (x == 66 ? x : (x == 67 ? x : (x == 68 ? x : (x == 69 ? x : (x == 70 ? x : (x == 71 ? x : (x == 72 ? x : (x == 73 ? x : (x == 74 ? x : (x == 75 ? x : (x == 76 ? x : (x == 77 ? x : (x == 78 ? x : (x == 79 ? x : (x == 80 ? x : (x == 81 ? x : (x == 82 ? x : (x == 83 ? x : (x == 84 ? x : (x == 85 ? x : (x == 86 ? x : (x == 87 ? x : (x == 88 ? x : (x == 89 ? x : (x == 90 ? x : (x == 91 ? x : (x == 92 ? x : (x == 93 ? x : (x == 94 ? x : (x == 95 ? x : (x == 96 ? x : (x == 97 ? x : (x == 98 ? x : (x == 99 ? x : (x == 100 ? x : (x == 101 ? x : (x == 102 ? x : (x == 103 ? x : (x == 104 ? x : (x == 105 ? x : (x == 106 ? x : (x == 107 ? x : (x == 108 ? x : (x == 109 ? x : (x == 110 ? x : (x == 111 ? x : (x == 112 ? x : (x == 113 ? x : (x == 114 ? x : (x == 115 ? x : (x == 116 ? x : (x == 117 ? x : (x == 118 ? x : (x == 119 ? x : (x == 120 ? x : (x == 121 ? x : (x == 122 ? x : (x == 123 ? x : (x == 124 ? x : (x == 125 ? x : (x == 126 ? x : (x == 127 ? x : (x == 128 ? x : (x == 129 ? x : (x == 130 ? x : (x == 131 ? x : (x == 132 ? x : (x == 133 ? x : (x == 134 ? x : (x == 135 ? x : (x == 136 ? x : (x == 137 ? x : (x == 138 ? x : (x == 139 ? x : (x == 140 ? x : (x == 141 ? x : (x == 142 ? x : (x == 143 ? x : (x == 144 ? x : (x == 145 ? x : (x == 146 ? x : (x == 147 ? x : (x == 148 ? x : (x == 149 ? x : (x == 150 ? x : (x == 151 ? x : (x == 152 ? x : (x == 153 ? x : (x == 154 ? x : (x == 155 ? x : (x == 156 ? x : (x == 157 ? x : (x == 158 ? x : (x == 159 ? x : (x == 160 ? x : (x == 161 ? x : (x == 162 ? x : (x == 163 ? x : (x == 164 ? x : (x == 165 ? x : (x == 166 ? x : (x == 167 ? x : (x == 168 ? x : (x == 169 ? x : (x == 170 ? x : (x == 171 ? x : (x == 172 ? x : (x == 173 ? x : (x == 174 ? x : (x == 175 ? x : (x == 176 ? x : (x == 177 ? x : (x == 178 ? x : (x == 179 ? x : (x == 180 ? x : (x == 181 ? x : (x == 182 ? x : (x == 183 ? x : (x == 184 ? x : (x == 185 ? x : (x == 186 ? x : (x == 187 ? x : (x == 188 ? x : (x == 189 ? x : (x == 190 ? x : (x == 191 ? x : (x == 192 ? x : (x == 193 ? x : (x == 194 ? x : (x == 195 ? x : (x == 196 ? x : (x == 197 ? x : (x == 198 ? x : (x == 199 ? x : (x == 200 ? x : (x == 201 ? x : (x == 202 ? x : (x == 203 ? x : (x == 204 ? x : (x == 205 ? x : (x == 206 ? x : (x == 207 ? x : (x == 208 ? x : (x == 209 ? x : (x == 210 ? x : (x == 211 ? x : (x == 212 ? x : (x == 213 ? x : (x == 214 ? x : (x == 215 ? x : (x == 216 ? x : (x == 217 ? x : (x == 218 ? x : (x == 219 ? x : (x == 220 ? x : (x == 221 ? x : (x == 222 ? x : (x == 223 ? x : (x == 224 ? x : (x == 225 ? x : (x == 226 ? x : (x == 227 ? x : (x == 228 ? x : (x == 229 ? x : (x == 230 ? x : (x == 231 ? x : (x == 232 ? x : (x == 233 ? x : (x == 234 ? x : (x == 235 ? x : (x == 236 ? x : (x == 237 ? x : (x == 238 ? x : (x == 239 ? x : (x == 240 ? x : (x == 241 ? x : (x == 242 ? x : (x == 243 ? x : (x == 244 ? x : (x == 245 ? x : (x == 246 ? x : (x == 247 ? x : (x == 248 ? x : (x == 249 ? x : (x == 250 ? x : (x == 251 ? x : (x == 252 ? x : (x == 253 ? x : (x == 254 ? x : (x == 255 ? x : 0)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));

tests/compiler/additional_benches/large_1.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type g8SH = u8;
 
 type V8fkp = u128;

tests/compiler/additional_benches/large_2.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type JzT3X = t5RQ;
 
 type RaEnk = JzT3X;

tests/compiler/additional_benches/large_3.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type f7emM = i32;
 
 type D7DUI = field;

tests/compiler/additional_benches/large_4.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type TESjC = u8;
 
 type XgJbM = ud9u;

tests/compiler/additional_benches/large_5.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type NmYjQ = i32;
 
 const K_baB = [85i8; 3];

tests/compiler/additional_benches/long_array.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 function main() {
     const arr1: [u8; (32, 32)] = [
         [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 31],

tests/compiler/additional_benches/long_expr.leo

@@ -0,0 +1,14 @@
+/*
+namespace: Bench
+expectation: Pass
+*/
+
+function main() -> u32 {
+    const a = 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32;
+    const b = 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32;
+    const c = 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32;
+    const d = 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32;
+    const e = 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32 + 1u32;
+
+    return a + b + c + d + e;
+}

tests/compiler/additional_benches/many_assigns.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 function main() {
     const x0: u8 = 0;
     const x1: u8 = x0;

tests/compiler/additional_benches/many_foos.leo

@@ -1,4 +1,9 @@
-function main() {
+/*
+namespace: Bench
+expectation: Pass
+*/
+
+function main() -> u8 {
     return x191(0u32);
 }
 

tests/compiler/additional_benches/massive.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type ipD5 = i8;
 
 type g52FJ = (field, u128);

tests/compiler/additional_benches/medium_1.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 const Hqyz7: address = aleo1nr8us4x0p4yyd99qa878e06jwddqr79gavznpxc6degn2gcv8syseq4dqu;
 
 

tests/compiler/additional_benches/medium_2.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type xs1w = i8;
 
 type Z5tB = field;

tests/compiler/additional_benches/medium_3.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type HGvc = char;
 
 type gOJKD = u64;

tests/compiler/additional_benches/medium_4.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type LWClh = u128;
 
 type IWJoY = i64;

tests/compiler/additional_benches/medium_5.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type pD0X5 = i8;
 
 type XKaI = i16;

tests/compiler/additional_benches/small_1.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type Ol3w = field;
 
 type Ahe2 = bool;

tests/compiler/additional_benches/small_2.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type uxXY = (u32, char);
 
 type gzOiV = bool;

tests/compiler/additional_benches/small_3.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type RoqL = i64;
 
 type AaPhh = u8;

tests/compiler/additional_benches/small_4.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type km8Fr = u32;
 
 type kzW6 = i32;

tests/compiler/additional_benches/small_5.leo

@@ -1,3 +1,8 @@
+/*
+namespace: Bench
+expectation: Skip
+*/
+
 type Rx6fk = u8;
 
 type Sq9N = i8;

tests/compiler/address/branch.leo

@@ -6,10 +6,10 @@ input_file: inputs/branch.in
 
 
 function main (x: address, y: bool) -> bool {
-    let z: address = aleo18cw5zdez3zhypev3tnfhmwvhre9ramwle4up947gcyy5rnmjw5yqn93wsr;
+    let z: address = aleo1fj982yqchhy973kz7e9jk6er7t6qd6jm9anplnlprem507w6lv9spwvfxx;
     if y {
-        z = aleo1f2gs8g0qpumlgzpvmkw3q07y6xrwsdr0lqsu9h9fgnh8d7e44v9qhpgpkj;
+        z = aleo16s003g206rjms5pm4ak48340f7y4z4dsskuqfrd2gvqz6umh2qfq7lajfp;
     }
 
-    return z == aleo1f2gs8g0qpumlgzpvmkw3q07y6xrwsdr0lqsu9h9fgnh8d7e44v9qhpgpkj;
+    return z == aleo1drcl2g8zxhxjzjw63ajp067gzvl94am3z7m7wgrzmr2ecd5sdq8sy66l5k;
 }

tests/compiler/address/equal.leo

@@ -7,7 +7,7 @@ input_file:
 */
 
 function main(x: address) -> bool {
-    const sender: address = aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8s7pyjh9;
+    const sender: address = aleo10qerras5799u6k7rjtc9y3hcwxuykr45qra7x7dp6jgnc0923czqm0lgta;
 
     return x == sender;
 }

tests/compiler/address/ternary.leo

@@ -7,8 +7,8 @@ input_file:
 */
 
 function main(x: address) -> bool {
-    const sender: address = aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8s7pyjh9;
+    const sender: address = aleo1l7ytv5jqjzpxtjqttl5z9mle8ujcpac9t6tkge5f4haah4pxas8sagzecd;
-    const receiver: address = aleo18qgam03qe483tdrcc3fkqwpp38ehff4a2xma6lu7hams6lfpgcpqy3sr3p;
+    const receiver: address = aleo1dtpkpg3d653mdlzh6g028937qdgujecn5gw5tzh7ftcvyz7jxvfqw6t8p6;
 
     return x == sender ? receiver == x : sender == x;
 }

tests/test-framework/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "leo-test-framework"
 version = "1.5.3"
-authors = [ "The Aleo Team <hello@aleo.org>" ]
+authors = ["The Aleo Team <hello@aleo.org>"]
 description = "Leo testing framework"
 homepage = "https://aleo.org"
 repository = "https://github.com/AleoHQ/leo"
@@ -10,62 +10,50 @@ keywords = [
   "cryptography",
   "leo",
   "programming-language",
-  "zero-knowledge"
+  "zero-knowledge",
 ]
-categories = [ "cryptography::cryptocurrencies", "web-programming" ]
+categories = ["cryptography::cryptocurrencies", "web-programming"]
-include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ]
+include = ["Cargo.toml", "src", "README.md", "LICENSE.md"]
 license = "GPL-3.0"
 edition = "2018"
 
+[[bench]]
+name = "leo_compiler"
+harness = false
+
+[dependencies]
+backtrace = "0.3.65"
+walkdir = "2.3.2"
+
 [dependencies.serde]
 version = "1.0"
-features = [ "derive" ]
+features = ["derive"]
 
 [dependencies.serde_json]
 version = "1.0"
-features = [ "preserve_order" ]
+features = ["preserve_order"]
 
 [dependencies.serde_yaml]
 version = "0.8"
 
-# List of dependencies for tgc binary;
-# disabled for now while porting modules from staging
-# [dependencies.leo-ast]
-# path = "../ast"
-# version = "1.5.2"
-
-# [dependencies.leo-ast-passes]
-# path = "../ast-passes"
-# version = "1.5.2"
-
-# [dependencies.leo-parser]
-# path = "../parser"
-# version = "1.5.2"
-
-# [dependencies.leo-imports]
-# path = "../imports"
-# version = "1.5.2"
-
-# [dependencies.leo-asg]
-# path = "../asg"
-# version = "1.5.2"
-
-# [dependencies.leo-compiler]
-# path = "../compiler"
-# version = "1.5.2"
-
-# [dependencies.structopt]
-# version = "0.3"
-
 [dependencies.clap]
 version = "3.2.1"
 features = ["derive"]
 
-# List of dependencies for errcov
-
 [dependencies.leo-errors]
 path = "../../leo/errors"
 version = "1.5.3"
 
 [dependencies.regex]
 version = "1.5"
+
+[dev-dependencies.criterion]
+version = "0.3"
+
+[dev-dependencies.leo-compiler]
+path = "../../compiler/compiler"
+version = "1.5.3"
+
+[dev-dependencies.leo-span]
+path = "../../leo/span"
+version = "1.5.3"

tests/test-framework/README.md

@@ -74,9 +74,32 @@ To do so you can simply remove the corresponding `.out` file in the `tests/expec
 
 To make several aspects of the test framework easier to work with there are several environment variables:
 
-- `TEST_FILTER` - runs all tests that contain the specified name.
+- `TEST_FILTER` - Now runs all tests in the given directory, or the exact given test.
-- `CLEAR_LEO_TEST_EXPECTATIONS` - which if set clears all current expectations and regenerates them all.
+  - `TEST_FILTER="address" cargo test -p leo-compiler` will run all tests in the located in `tests/compiler/address`.
+  - `TEST_FILTER="address/branch.leo" cargo test -p leo-compiler` will run the test located in `tests/compiler/address/branch.leo`.
+- `CLEAR_LEO_TEST_EXPECTATIONS` - which if set clears all current expectations for the tests being run and regenerates them all.
 
 To set environment variables please look at your Shell(bash/powershell/cmd/fish/etc) specific implementation for doing so
 
 **NOTE**: Don't forget to clear the environment variable after running it with that setting, or set a temporary env variable if your shell supports it.
+
+### Benchmarking
+
+The test-framework is now used to easily benchmark Leo, by running on all compiler tests that have the `Pass` expectation.
+Additionally, you can create additional benchmark tests by adding them in the test directory by giving them the namespace of `Bench`.
+
+#### Running
+
+There are currently four different kinds of benchmarks to run:
+
+- parse - benchmarks parsing of Leo files.
+- symbol - benchmarks the symbol table generation pass.
+- type - benchmarks the type checking pass.
+- full - benchmarks all aspects of compilation.
+
+To run the benchmarks the command is `cargo bench -p leo-test-framework`.
+This by default runs all the above-mentioned benchmark suites.
+To specify a specific one you would do `cargo bench -p leo-test-framework parse` or any of the above-listed benchmark suites.
+
+**NOTE** Benchmarks are affected by the `TEST_FILTER` environment variable.
+They are also machine dependent on your pc and are impacted by other open applications.

tests/test-framework/benches/leo_compiler.rs

@@ -0,0 +1,228 @@
+// Copyright (C) 2019-2022 Aleo Systems Inc.
+// This file is part of the Leo library.
+
+// The Leo library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// The Leo library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
+
+//! This file contains tools for benchmarking the Leo compiler and its stages.
+
+use leo_compiler::Compiler;
+use leo_errors::emitter::{Emitter, Handler};
+use leo_span::{
+    source_map::FileName,
+    symbol::{SessionGlobals, SESSION_GLOBALS},
+};
+use leo_test_framework::get_benches;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use std::{
+    path::PathBuf,
+    time::{Duration, Instant},
+};
+
+/// An enum to represent the stage of the Compiler we are benchmarking.
+enum BenchMode {
+    /// Benchmarks parsing.
+    Parse,
+    /// Benchmarks symbol table generation.
+    Symbol,
+    /// Benchmarks type checking.
+    Type,
+    /// Benchmarks all the above stages.
+    Full,
+}
+
+/// A dummy buffer emitter since we only test on valid programs.
+struct BufEmitter;
+
+impl Emitter for BufEmitter {
+    fn emit_err(&mut self, _: leo_errors::LeoError) {}
+
+    fn last_emitted_err_code(&self) -> Option<i32> {
+        None
+    }
+
+    fn emit_warning(&mut self, _: leo_errors::LeoWarning) {}
+}
+
+impl BufEmitter {
+    fn new_handler() -> Handler {
+        Handler::new(Box::new(Self))
+    }
+}
+
+/// The name of the test, and the test content.
+#[derive(Clone)]
+struct Sample {
+    name: String,
+    input: String,
+}
+
+/// A helper function to help create a Leo Compiler struct.
+fn new_compiler(handler: &Handler) -> Compiler<'_> {
+    Compiler::new(
+        handler,
+        PathBuf::from(String::new()),
+        PathBuf::from(String::new()),
+        None,
+    )
+}
+
+impl Sample {
+    /// Loads all the benchmark samples.
+    /// Leverages the test-framework to grab all tests
+    /// that are passing compiler tests or marked as benchmark tests.
+    fn load_samples() -> Vec<Self> {
+        get_benches()
+            .into_iter()
+            .map(|(name, input)| Self { name, input })
+            .collect()
+    }
+
+    fn data(&self) -> (&str, FileName) {
+        black_box((&self.input, FileName::Custom(String::new())))
+    }
+
+    fn bench(&self, c: &mut Criterion, mode: BenchMode) {
+        match mode {
+            BenchMode::Parse => self.bench_parse(c),
+            BenchMode::Symbol => self.bench_symbol_table(c),
+            BenchMode::Type => self.bench_type_checker(c),
+            BenchMode::Full => self.bench_full(c),
+        }
+    }
+
+    fn bench_parse(&self, c: &mut Criterion) {
+        c.bench_function(&format!("parse {}", self.name), |b| {
+            // Iter custom is used so we can use custom timings around the compiler stages.
+            // This way we can only time the necessary stage.
+            b.iter_custom(|iters| {
+                let mut time = Duration::default();
+                for _ in 0..iters {
+                    SESSION_GLOBALS.set(&SessionGlobals::default(), || {
+                        let handler = BufEmitter::new_handler();
+                        let mut compiler = new_compiler(&handler);
+                        let (input, name) = self.data();
+                        let start = Instant::now();
+                        let out = compiler.parse_program_from_string(input, name);
+                        time += start.elapsed();
+                        out.expect("Failed to parse program")
+                    });
+                }
+                time
+            })
+        });
+    }
+
+    fn bench_symbol_table(&self, c: &mut Criterion) {
+        c.bench_function(&format!("symbol table pass {}", self.name), |b| {
+            // Iter custom is used so we can use custom timings around the compiler stages.
+            // This way we can only time the necessary stage.
+            b.iter_custom(|iters| {
+                let mut time = Duration::default();
+                for _ in 0..iters {
+                    SESSION_GLOBALS.set(&SessionGlobals::default(), || {
+                        let handler = BufEmitter::new_handler();
+                        let mut compiler = new_compiler(&handler);
+                        let (input, name) = self.data();
+                        compiler
+                            .parse_program_from_string(input, name)
+                            .expect("Failed to parse program");
+                        let start = Instant::now();
+                        let out = compiler.symbol_table_pass();
+                        time += start.elapsed();
+                        out.expect("failed to generate symbol table");
+                    });
+                }
+                time
+            })
+        });
+    }
+
+    fn bench_type_checker(&self, c: &mut Criterion) {
+        c.bench_function(&format!("type checker pass {}", self.name), |b| {
+            // Iter custom is used so we can use custom timings around the compiler stages.
+            // This way we can only time the necessary stage.
+            b.iter_custom(|iters| {
+                let mut time = Duration::default();
+                for _ in 0..iters {
+                    SESSION_GLOBALS.set(&SessionGlobals::default(), || {
+                        let handler = BufEmitter::new_handler();
+                        let mut compiler = new_compiler(&handler);
+                        let (input, name) = self.data();
+                        compiler
+                            .parse_program_from_string(input, name)
+                            .expect("Failed to parse program");
+                        let mut symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table");
+                        let start = Instant::now();
+                        let out = compiler.type_checker_pass(&mut symbol_table);
+                        time += start.elapsed();
+                        out.expect("failed to run type check pass")
+                    });
+                }
+                time
+            })
+        });
+    }
+
+    fn bench_full(&self, c: &mut Criterion) {
+        c.bench_function(&format!("full {}", self.name), |b| {
+            // Iter custom is used so we can use custom timings around the compiler stages.
+            // This way we can only time the necessary stages.
+            b.iter_custom(|iters| {
+                let mut time = Duration::default();
+                for _ in 0..iters {
+                    SESSION_GLOBALS.set(&SessionGlobals::default(), || {
+                        let handler = BufEmitter::new_handler();
+                        let mut compiler = new_compiler(&handler);
+                        let (input, name) = self.data();
+                        let start = Instant::now();
+                        compiler
+                            .parse_program_from_string(input, name)
+                            .expect("Failed to parse program");
+                        let mut symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table");
+                        compiler
+                            .type_checker_pass(&mut symbol_table)
+                            .expect("failed to run type check pass");
+                        time += start.elapsed();
+                    });
+                }
+                time
+            })
+        });
+    }
+}
+
+macro_rules! bench {
+    ($name:ident, $mode:expr) => {
+        fn $name(c: &mut Criterion) {
+            Sample::load_samples().into_iter().for_each(|s| s.bench(c, $mode))
+        }
+    };
+}
+
+bench!(bench_parse, BenchMode::Parse);
+bench!(bench_symbol, BenchMode::Symbol);
+bench!(bench_type, BenchMode::Type);
+bench!(bench_full, BenchMode::Full);
+
+criterion_group!(
+    name = benches;
+    config = Criterion::default().sample_size(200).measurement_time(Duration::from_secs(10)).nresamples(200_000);
+    targets =
+        bench_parse,
+        bench_symbol,
+        bench_type,
+        bench_full
+);
+criterion_main!(benches);

tests/test-framework/src/bin/errcov.rs

@@ -1,239 +1,241 @@
-// Copyright (C) 2019-2021 Aleo Systems Inc.
+// // Copyright (C) 2019-2021 Aleo Systems Inc.
-// This file is part of the Leo library.
+// // This file is part of the Leo library.
 
-// The Leo library is free software: you can redistribute it and/or modify
+// // The Leo library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
+// // it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
+// // the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
+// // (at your option) any later version.
 
-// The Leo library is distributed in the hope that it will be useful,
+// // The Leo library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// // but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
+// // GNU General Public License for more details.
 
-// You should have received a copy of the GNU General Public License
+// // You should have received a copy of the GNU General Public License
-// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
+// // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
-use leo_errors::{AstError, InputError, LeoMessageCode, PackageError, ParserError};
+// use leo_errors::{AstError, InputError, LeoMessageCode, PackageError, ParserError};
-use leo_test_framework::{
+// use leo_test_framework::{
-    fetch::find_tests,
+//     fetch::find_tests,
-    output::TestExpectation,
+//     output::TestExpectation,
-    test::{extract_test_config, TestExpectationMode as Expectation},
+//     test::{extract_test_config, TestExpectationMode as Expectation},
-};
+// };
 
-use clap::StructOpt;
+// use regex::Regex;
-use regex::Regex;
+// use serde_yaml::Value;
-use serde_yaml::Value;
+// use std::collections::{BTreeMap, HashSet};
-use std::collections::{BTreeMap, HashSet};
+// use std::{error::Error, fs, io, path::PathBuf};
-use std::{error::Error, fs, io, path::PathBuf};
+// use structopt::{clap::AppSettings, StructOpt};
 
-#[derive(StructOpt)]
+// #[derive(StructOpt)]
-#[structopt(name = "error-coverage", author = "The Aleo Team <hello@aleo.org>")]
+// #[structopt(name = "error-coverage", author = "The Aleo Team <hello@aleo.org>", setting = AppSettings::ColoredHelp)]
-struct Opt {
+// struct Opt {
-    #[structopt(
+//     #[structopt(
-        short,
+//         short,
-        long,
+//         long,
-        help = "Path to the output file, defaults to stdout.",
+//         help = "Path to the output file, defaults to stdout.",
-        parse(from_os_str)
+//         parse(from_os_str)
-    )]
+//     )]
-    output: Option<PathBuf>,
+//     output: Option<PathBuf>,
-}
+// }
 
-fn main() {
+// fn main() {
-    handle_error(run_with_args(Opt::parse()));
+//     handle_error(run_with_args(Opt::from_args()));
-}
+// }
 
-fn run_with_args(opt: Opt) -> Result<(), Box<dyn Error>> {
+// fn run_with_args(opt: Opt) -> Result<(), Box<dyn Error>> {
-    // Variable that stores all the tests.
+//     // Variable that stores all the tests.
-    let mut tests = Vec::new();
+//     let mut tests = Vec::new();
-    let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+//     let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    test_dir.push("../");
+//     test_dir.push("../");
 
-    let mut expectation_dir = test_dir.clone();
+//     let mut expectation_dir = test_dir.clone();
-    expectation_dir.push("expectations");
+//     expectation_dir.push("expectations");
 
-    find_tests(&test_dir, &mut tests);
+//     find_tests(&test_dir, &mut tests);
 
-    // Store all covered error codes
+//     // Store all covered error codes
-    let mut found_codes = BTreeMap::new();
+//     let mut found_codes = BTreeMap::new();
-    let re = Regex::new(r"Error \[(?P<code>.*)\]:.*").unwrap();
+//     let re = Regex::new(r"Error \[(?P<code>.*)\]:.*").unwrap();
 
-    for (path, content) in tests.into_iter() {
+//     for (path, content) in tests.into_iter() {
-        if let Some(config) = extract_test_config(&content) {
+//         if let Some(config) = extract_test_config(&content) {
-            // Skip passing tests.
+//             // Skip passing tests.
-            if config.expectation == Expectation::Pass {
+//             if config.expectation == Expectation::Pass {
-                continue;
+//                 continue;
-            }
+//             }
 
-            let mut expectation_path = expectation_dir.clone();
+//             let mut expectation_path = expectation_dir.clone();
 
-            let path = PathBuf::from(path);
+//             let path = PathBuf::from(path);
-            let relative_path = path.strip_prefix(&test_dir).expect("path error for test");
+//             let relative_path = path.strip_prefix(&test_dir).expect("path error for test");
 
-            let mut relative_expectation_path = relative_path.to_str().unwrap().to_string();
+//             let mut relative_expectation_path = relative_path.to_str().unwrap().to_string();
-            relative_expectation_path += ".out";
+//             relative_expectation_path += ".out";
 
-            // Add expectation category
+//             // Add expectation category
-            if relative_expectation_path.starts_with("compiler") {
+//             if relative_expectation_path.starts_with("compiler") {
-                expectation_path.push("compiler");
+//                 expectation_path.push("compiler");
-            } else {
+//             } else {
-                expectation_path.push("parser");
+//                 expectation_path.push("parser");
-            }
+//             }
-            expectation_path.push(&relative_expectation_path);
+//             expectation_path.push(&relative_expectation_path);
 
-            if expectation_path.exists() {
+//             if expectation_path.exists() {
-                let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file");
+//                 let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file");
-                let expectation: TestExpectation =
+//                 let expectation: TestExpectation =
-                    serde_yaml::from_str(&raw).expect("invalid yaml in expectations file");
+//                     serde_yaml::from_str(&raw).expect("invalid yaml in expectations file");
 
-                for value in expectation.outputs {
+//                 for value in expectation.outputs {
-                    if let serde_yaml::Value::String(message) = value {
+//                     if let serde_yaml::Value::String(message) = value {
-                        if let Some(caps) = re.captures(&message) {
+//                         if let Some(caps) = re.captures(&message) {
-                            if let Some(code) = caps.name("code") {
+//                             if let Some(code) = caps.name("code") {
-                                let files = found_codes
+//                                 let files = found_codes
-                                    .entry(code.as_str().to_string())
+//                                     .entry(code.as_str().to_string())
-                                    .or_insert_with(HashSet::new);
+//                                     .or_insert_with(HashSet::new);
-                                let path = expectation_path
+//                                 let path = expectation_path
-                                    .strip_prefix(test_dir.clone())
+//                                     .strip_prefix(test_dir.clone())
-                                    .expect("invalid prefix for expectation path");
+//                                     .expect("invalid prefix for expectation path");
-                                files.insert(PathBuf::from(path));
+//                                 files.insert(PathBuf::from(path));
-                            }
+//                             }
-                        }
+//                         }
-                    }
+//                     }
-                }
+//                 }
-            }
+//             }
-        }
+//         }
-    }
+//     }
 
-    // Collect all defined error codes.
+//     // Collect all defined error codes.
-    let mut all_codes = HashSet::new();
+//     let mut all_codes = HashSet::new();
-    collect_error_codes(
+//     collect_error_codes(
-        &mut all_codes,
+//         &mut all_codes,
-        AstError::message_type(),
+//         AstError::message_type(),
-        AstError::code_identifier(),
+//         AstError::code_identifier(),
-        AstError::code_mask(),
+//         AstError::code_mask(),
-        AstError::num_exit_codes(),
+//         AstError::num_exit_codes(),
-    );
+//     );
-    collect_error_codes(
+//     collect_error_codes(
-        &mut all_codes,
+//         &mut all_codes,
-        InputError::message_type(),
+//         InputError::message_type(),
-        InputError::code_identifier(),
+//         InputError::code_identifier(),
-        InputError::code_mask(),
+//         InputError::code_mask(),
-        InputError::num_exit_codes(),
+//         InputError::num_exit_codes(),
-    );
+//     );
-    collect_error_codes(
+//     collect_error_codes(
-        &mut all_codes,
+//         &mut all_codes,
-        PackageError::message_type(),
+//         PackageError::message_type(),
-        PackageError::code_identifier(),
+//         PackageError::code_identifier(),
-        PackageError::code_mask(),
+//         PackageError::code_mask(),
-        PackageError::num_exit_codes(),
+//         PackageError::num_exit_codes(),
-    );
+//     );
-    collect_error_codes(
+//     collect_error_codes(
-        &mut all_codes,
+//         &mut all_codes,
-        ParserError::message_type(),
+//         ParserError::message_type(),
-        ParserError::code_identifier(),
+//         ParserError::code_identifier(),
-        ParserError::code_mask(),
+//         ParserError::code_mask(),
-        ParserError::num_exit_codes(),
+//         ParserError::num_exit_codes(),
-    );
+//     );
 
-    // Repackage data into values compatible with serde_yaml
+//     // Repackage data into values compatible with serde_yaml
-    let mut covered_errors = serde_yaml::Mapping::new();
+//     let mut covered_errors = serde_yaml::Mapping::new();
-    let mut unknown_errors = serde_yaml::Mapping::new();
+//     let mut unknown_errors = serde_yaml::Mapping::new();
 
-    for (code, paths) in found_codes.iter() {
+//     for (code, paths) in found_codes.iter() {
-        let mut yaml_paths = Vec::with_capacity(paths.len());
+//         let mut yaml_paths = Vec::with_capacity(paths.len());
-        for path in paths {
+//         for path in paths {
-            yaml_paths.push(path.to_str().unwrap());
+//             yaml_paths.push(path.to_str().unwrap());
-        }
+//         }
-        yaml_paths.sort_unstable();
+//         yaml_paths.sort_unstable();
-        let yaml_paths = yaml_paths.iter().map(|s| Value::String(s.to_string())).collect();
+//         let yaml_paths = yaml_paths.iter().map(|s| Value::String(s.to_string())).collect();
 
-        if all_codes.contains(code) {
+//         if all_codes.contains(code) {
-            covered_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths));
+//             covered_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths));
-        } else {
+//         } else {
-            unknown_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths));
+//             unknown_errors.insert(Value::String(code.to_owned()), Value::Sequence(yaml_paths));
-        }
+//         }
-        all_codes.remove(code);
+//         all_codes.remove(code);
-    }
+//     }
 
-    let mut codes: Vec<String> = all_codes.drain().collect();
+//     let mut codes: Vec<String> = all_codes.drain().collect();
-    codes.sort();
+//     codes.sort();
 
-    let mut uncovered_errors = Vec::new();
+//     let mut uncovered_errors = Vec::new();
-    for code in codes {
+//     for code in codes {
-        uncovered_errors.push(Value::String(code))
+//         uncovered_errors.push(Value::String(code))
-    }
+//     }
 
-    let mut uncovered_information = serde_yaml::Mapping::new();
+//     let mut uncovered_information = serde_yaml::Mapping::new();
-    uncovered_information.insert(
+//     uncovered_information.insert(
-        Value::String(String::from("count")),
+//         Value::String(String::from("count")),
-        Value::Number(serde_yaml::Number::from(uncovered_errors.len())),
+//         Value::Number(serde_yaml::Number::from(uncovered_errors.len())),
-    );
+//     );
-    uncovered_information.insert(Value::String(String::from("codes")), Value::Sequence(uncovered_errors));
+//     uncovered_information.insert(Value::String(String::from("codes")), Value::Sequence(uncovered_errors));
 
-    let mut covered_information = serde_yaml::Mapping::new();
+//     let mut covered_information = serde_yaml::Mapping::new();
-    covered_information.insert(
+//     covered_information.insert(
-        Value::String(String::from("count")),
+//         Value::String(String::from("count")),
-        Value::Number(serde_yaml::Number::from(covered_errors.len())),
+//         Value::Number(serde_yaml::Number::from(covered_errors.len())),
-    );
+//     );
-    covered_information.insert(Value::String(String::from("codes")), Value::Mapping(covered_errors));
+//     covered_information.insert(Value::String(String::from("codes")), Value::Mapping(covered_errors));
 
-    let mut unknown_information = serde_yaml::Mapping::new();
+//     let mut unknown_information = serde_yaml::Mapping::new();
-    unknown_information.insert(
+//     unknown_information.insert(
-        Value::String(String::from("count")),
+//         Value::String(String::from("count")),
-        Value::Number(serde_yaml::Number::from(unknown_errors.len())),
+//         Value::Number(serde_yaml::Number::from(unknown_errors.len())),
-    );
+//     );
-    unknown_information.insert(Value::String(String::from("codes")), Value::Mapping(unknown_errors));
+//     unknown_information.insert(Value::String(String::from("codes")), Value::Mapping(unknown_errors));
 
-    let mut results = serde_yaml::Mapping::new();
+//     let mut results = serde_yaml::Mapping::new();
-    results.insert(
+//     results.insert(
-        Value::String(String::from("uncovered")),
+//         Value::String(String::from("uncovered")),
-        Value::Mapping(uncovered_information),
+//         Value::Mapping(uncovered_information),
-    );
+//     );
 
-    results.insert(
+//     results.insert(
-        Value::String(String::from("covered")),
+//         Value::String(String::from("covered")),
-        Value::Mapping(covered_information),
+//         Value::Mapping(covered_information),
-    );
+//     );
-    results.insert(
+//     results.insert(
-        Value::String(String::from("unknown")),
+//         Value::String(String::from("unknown")),
-        Value::Mapping(unknown_information),
+//         Value::Mapping(unknown_information),
-    );
+//     );
 
-    // Output error coverage results
+//     // Output error coverage results
-    if let Some(pathbuf) = opt.output {
+//     if let Some(pathbuf) = opt.output {
-        let file = fs::File::create(pathbuf).expect("error creating output file");
+//         let file = fs::File::create(pathbuf).expect("error creating output file");
-        serde_yaml::to_writer(file, &results).expect("serialization failed for error coverage report");
+//         serde_yaml::to_writer(file, &results).expect("serialization failed for error coverage report");
-    } else {
+//     } else {
-        serde_yaml::to_writer(io::stdout(), &results).expect("serialization failed for error coverage report");
+//         serde_yaml::to_writer(io::stdout(), &results).expect("serialization failed for error coverage report");
-    }
+//     }
 
-    Ok(())
+//     Ok(())
-}
+// }
 
-fn collect_error_codes(
+// fn collect_error_codes(
-    codes: &mut HashSet<String>,
+//     codes: &mut HashSet<String>,
-    error_type: String,
+//     error_type: String,
-    code_identifier: i8,
+//     code_identifier: i8,
-    exit_code_mask: i32,
+//     exit_code_mask: i32,
-    num_exit_codes: i32,
+//     num_exit_codes: i32,
-) {
+// ) {
-    for exit_code in 0..num_exit_codes {
+//     for exit_code in 0..num_exit_codes {
-        codes.insert(format!(
+//         codes.insert(format!(
-            "E{}{:0>3}{:0>4}",
+//             "E{}{:0>3}{:0>4}",
-            error_type,
+//             error_type,
-            code_identifier,
+//             code_identifier,
-            exit_code_mask + exit_code,
+//             exit_code_mask + exit_code,
-        ));
+//         ));
-    }
+//     }
-}
+// }
 
-fn handle_error(res: Result<(), Box<dyn Error>>) {
+// fn handle_error(res: Result<(), Box<dyn Error>>) {
-    match res {
+//     match res {
-        Ok(_) => (),
+//         Ok(_) => (),
-        Err(err) => {
+//         Err(err) => {
-            eprintln!("Error: {}", err);
+//             eprintln!("Error: {}", err);
-            std::process::exit(1);
+//             std::process::exit(1);
-        }
+//         }
-    }
+//     }
-}
+// }
+
+fn main() {}

tests/test-framework/src/error.rs

@@ -177,5 +177,6 @@ pub fn emit_errors(
             }
             None
         }
+        (Ok(_), TestExpectationMode::Skip) => None,
     }
 }

tests/test-framework/src/fetch.rs

@@ -14,27 +14,30 @@
 // You should have received a copy of the GNU General Public License
 // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
-use std::{fs, path::Path};
+use std::{
+    fs,
+    path::{Path, PathBuf},
+};
 
-pub fn find_tests<T: AsRef<Path>>(path: T, out: &mut Vec<(String, String)>) {
+use walkdir::WalkDir;
-    for entry in fs::read_dir(path).expect("fail to read tests") {
+
-        let entry = entry.expect("fail to read tests").path();
+pub fn find_tests(path: &Path) -> impl Iterator<Item = (PathBuf, String)> {
-        if entry.is_dir() {
+    WalkDir::new(path).into_iter().flatten().filter_map(move |f| {
-            find_tests(entry.as_path(), out);
+        let path = f.path();
-            continue;
+        path.extension().filter(|s| *s == "leo").map(|_| {
-        } else if entry.extension().and_then(|x| x.to_str()).unwrap_or_default() != "leo" {
+            (
-            continue;
+                path.to_path_buf(),
-        }
+                fs::read_to_string(path).expect("failed to read test"),
-        let content = fs::read_to_string(entry.as_path()).expect("failed to read test");
+            )
-        out.push((entry.as_path().to_str().unwrap_or_default().to_string(), content));
+        })
-    }
+    })
 }
 
-pub fn split_tests_oneline(source: &str) -> Vec<&str> {
+pub fn split_tests_one_line(source: &str) -> Vec<&str> {
     source.lines().map(|x| x.trim()).filter(|x| !x.is_empty()).collect()
 }
 
-pub fn split_tests_twoline(source: &str) -> Vec<String> {
+pub fn split_tests_two_line(source: &str) -> Vec<String> {
     let mut out = vec![];
     let mut lines = vec![];
     for line in source.lines() {

tests/test-framework/src/runner.rs

@@ -50,6 +50,10 @@ pub trait Runner {
     fn resolve_namespace(&self, name: &str) -> Option<Box<dyn Namespace>>;
 }
 
+fn is_env_var_set(var: &str) -> bool {
+    std::env::var(var).unwrap_or_else(|_| "".to_string()).trim().is_empty()
+}
+
 fn set_hook() -> Arc<Mutex<Option<String>>> {
     let panic_buf = Arc::new(Mutex::new(None));
     let thread_id = thread::current().id();
@@ -57,7 +61,11 @@ fn set_hook() -> Arc<Mutex<Option<String>>> {
         let panic_buf = panic_buf.clone();
         Box::new(move |e| {
             if thread::current().id() == thread_id {
-                *panic_buf.lock().unwrap() = Some(e.to_string());
+                if !is_env_var_set("RUST_BACKTRACE") {
+                    *panic_buf.lock().unwrap() = Some(format!("{:?}", backtrace::Backtrace::new()));
+                } else {
+                    *panic_buf.lock().unwrap() = Some(e.to_string());
+                }
             } else {
                 println!("{}", e)
             }
@@ -74,93 +82,132 @@ fn take_hook(
     output.map_err(|_| panic_buf.lock().unwrap().take().expect("failed to get panic message"))
 }
 
+pub struct TestCases {
+    tests: Vec<(PathBuf, String)>,
+    path_prefix: PathBuf,
+    fail_categories: Vec<TestFailure>,
+}
+
+impl TestCases {
+    fn new(expectation_category: &str, additional_check: impl Fn(&TestConfig) -> bool) -> (Self, Vec<TestConfig>) {
+        let mut path_prefix = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+        path_prefix.push("../../tests/");
+        path_prefix.push(expectation_category);
+        if let Ok(p) = std::env::var("TEST_FILTER") {
+            path_prefix.push(p);
+        }
+
+        let mut expectation_dir = path_prefix.clone();
+        expectation_dir.push("expectations");
+
+        let mut new = Self {
+            tests: Vec::new(),
+            path_prefix,
+            fail_categories: Vec::new(),
+        };
+        let tests = new.load_tests(additional_check);
+        (new, tests)
+    }
+
+    fn load_tests(&mut self, additional_check: impl Fn(&TestConfig) -> bool) -> Vec<TestConfig> {
+        let mut configs = Vec::new();
+
+        self.tests = find_tests(&self.path_prefix.clone())
+            .filter(|(path, content)| {
+                let config = match extract_test_config(content) {
+                    None => {
+                        self.fail_categories.push(TestFailure {
+                            path: path.to_str().unwrap_or("").to_string(),
+                            errors: vec![TestError::MissingTestConfig],
+                        });
+                        return true;
+                    }
+                    Some(cfg) => cfg,
+                };
+
+                let res = additional_check(&config);
+                configs.push(config);
+                res
+            })
+            .collect();
+
+        configs
+    }
+
+    pub(crate) fn process_tests<P, O>(&mut self, configs: Vec<TestConfig>, mut process: P) -> Vec<O>
+    where
+        P: FnMut(&mut Self, (&Path, &str, &str, TestConfig)) -> O,
+    {
+        std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files
+        std::env::set_var("LEO_TESTFRAMEWORK", "true");
+
+        let mut output = Vec::new();
+        for ((path, content), config) in self.tests.clone().iter().zip(configs.into_iter()) {
+            let test_name = path
+                .file_stem()
+                .expect("no file name for test")
+                .to_str()
+                .unwrap()
+                .to_string();
+
+            let end_of_header = content.find("*/").expect("failed to find header block in test");
+            let content = &content[end_of_header + 2..];
+
+            output.push(process(self, (path, content, &test_name, config)));
+
+            std::env::remove_var("LEO_TESTFRAMEWORK");
+        }
+        output
+    }
+
+    fn clear_expectations(&self, path: &Path, expectation_category: &str) -> (PathBuf, Option<TestExpectation>) {
+        let test_dir = [env!("CARGO_MANIFEST_DIR"), "../../tests/"].iter().collect::<PathBuf>();
+        let relative_path = path.strip_prefix(&test_dir).expect("path error for test");
+        let expectation_path = test_dir
+            .join("expectations")
+            .join(expectation_category)
+            .join(relative_path.parent().expect("no parent dir for test"))
+            .join(relative_path.file_name().expect("no file name for test"))
+            .with_extension("out");
+
+        if expectation_path.exists() {
+            if !is_env_var_set("CLEAR_LEO_TEST_EXPECTATIONS") {
+                (expectation_path, None)
+            } else {
+                let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file");
+                (
+                    expectation_path,
+                    Some(serde_yaml::from_str(&raw).expect("invalid yaml in expectations file")),
+                )
+            }
+        } else {
+            (expectation_path, None)
+        }
+    }
+}
+
 pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
-    std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files
+    let (mut cases, configs) = TestCases::new(expectation_category, |_| true);
-    std::env::set_var("LEO_TESTFRAMEWORK", "true");
+
     let mut pass_categories = 0;
     let mut pass_tests = 0;
     let mut fail_tests = 0;
-    let mut fail_categories = Vec::new();
-
-    let mut tests = Vec::new();
-    let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
-    test_dir.push("../../tests/");
-
-    let mut expectation_dir = test_dir.clone();
-    expectation_dir.push("expectations");
-
-    find_tests(&test_dir, &mut tests);
-
-    let filter = std::env::var("TEST_FILTER").unwrap_or_default();
-    let filter = filter.trim();
 
     let mut outputs = vec![];
-
+    cases.process_tests(configs, |cases, (path, content, test_name, config)| {
-    for (path, content) in tests.into_iter() {
+        let namespace = match runner.resolve_namespace(&config.namespace) {
-        if !filter.is_empty() && !path.contains(filter) {
+            Some(ns) => ns,
-            continue;
+            None => return,
-        }
-        let config = extract_test_config(&content);
-        if config.is_none() {
-            //panic!("missing configuration for {}", path);
-            // fail_categories.push(TestFailure {
-            //     path,
-            //     errors: vec![TestError::MissingTestConfig],
-            // });
-            continue;
-        }
-        let config = config.unwrap();
-        let namespace = runner.resolve_namespace(&config.namespace);
-        if namespace.is_none() {
-            continue;
-        }
-        let namespace = namespace.unwrap();
-
-        let path = Path::new(&path);
-        let relative_path = path.strip_prefix(&test_dir).expect("path error for test");
-        let mut expectation_path = expectation_dir.clone();
-        expectation_path.push(expectation_category);
-        expectation_path.push(relative_path.parent().expect("no parent dir for test"));
-        let mut expectation_name = relative_path
-            .file_name()
-            .expect("no file name for test")
-            .to_str()
-            .unwrap()
-            .to_string();
-        expectation_name += ".out";
-        expectation_path.push(&expectation_name);
-
-        let test_name = relative_path
-            .file_stem()
-            .expect("no file name for test")
-            .to_str()
-            .unwrap()
-            .to_string();
-
-        let expectations: Option<TestExpectation> = if expectation_path.exists() {
-            if !std::env::var("CLEAR_LEO_TEST_EXPECTATIONS")
-                .unwrap_or_default()
-                .trim()
-                .is_empty()
-            {
-                None
-            } else {
-                let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file");
-                Some(serde_yaml::from_str(&raw).expect("invalid yaml in expectations file"))
-            }
-        } else {
-            None
         };
 
-        let end_of_header = content.find("*/").expect("failed to find header block in test");
+        let (expectation_path, expectations) = cases.clear_expectations(path, expectation_category);
-        let content = &content[end_of_header + 2..];
 
         let tests = match namespace.parse_type() {
-            ParseType::Line => crate::fetch::split_tests_oneline(content)
+            ParseType::Line => crate::fetch::split_tests_one_line(content)
                 .into_iter()
                 .map(|x| x.to_string())
                 .collect(),
-            ParseType::ContinuousLines => crate::fetch::split_tests_twoline(content),
+            ParseType::ContinuousLines => crate::fetch::split_tests_two_line(content),
             ParseType::Whole => vec![content.to_string()],
         };
 
@@ -172,7 +219,6 @@ pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
         }
 
         let mut new_outputs = vec![];
-
         let mut expected_output = expectations.as_ref().map(|x| x.outputs.iter());
         for (i, test) in tests.into_iter().enumerate() {
             let expected_output = expected_output.as_mut().and_then(|x| x.next()).cloned();
@@ -180,7 +226,7 @@ pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
             let panic_buf = set_hook();
             let leo_output = panic::catch_unwind(|| {
                 namespace.run_test(Test {
-                    name: test_name.clone(),
+                    name: test_name.to_string(),
                     content: test.clone(),
                     path: path.into(),
                     config: config.extra.clone(),
@@ -215,47 +261,26 @@ pub fn run_tests<T: Runner>(runner: &T, expectation_category: &str) {
             }
             pass_categories += 1;
         } else {
-            fail_categories.push(TestFailure {
+            cases.fail_categories.push(TestFailure {
                 path: path.to_str().unwrap().to_string(),
                 errors,
             })
         }
-    }
+    });
-    if !fail_categories.is_empty() {
+}
-        for (i, fail) in fail_categories.iter().enumerate() {
+
-            println!(
+/// returns (name, content) for all benchmark samples
-                "\n\n-----------------TEST #{} FAILED (and shouldn't have)-----------------",
+pub fn get_benches() -> Vec<(String, String)> {
-                i + 1
+    let (mut cases, configs) = TestCases::new("compiler", |config| {
-            );
+        (&config.namespace == "Bench" && config.expectation == TestExpectationMode::Pass)
-            println!("File: {}", fail.path);
+            || (&config.namespace == "Compile"
-            for error in &fail.errors {
+                && !matches!(
-                println!("{}", error);
+                    config.expectation,
-            }
+                    TestExpectationMode::Fail | TestExpectationMode::Skip
-        }
+                ))
-        panic!(
+    });
-            "failed {}/{} tests in {}/{} categories",
+
-            pass_tests,
+    cases.process_tests(configs, |_, (_, content, test_name, _)| {
-            fail_tests + pass_tests,
+        (test_name.to_string(), content.to_string())
-            fail_categories.len(),
+    })
-            fail_categories.len() + pass_categories
-        );
-    } else {
-        for (path, new_expectation) in outputs {
-            std::fs::create_dir_all(path.parent().unwrap()).expect("failed to make test expectation parent directory");
-            std::fs::write(
-                &path,
-                serde_yaml::to_string(&new_expectation).expect("failed to serialize expectation yaml"),
-            )
-            .expect("failed to write expectation file");
-        }
-        println!(
-            "passed {}/{} tests in {}/{} categories",
-            pass_tests,
-            fail_tests + pass_tests,
-            pass_categories,
-            pass_categories
-        );
-    }
-
-    std::env::remove_var("LEO_TESTFRAMEWORK");
 }

tests/test-framework/src/test.rs

@@ -20,6 +20,7 @@ use std::collections::BTreeMap;
 pub enum TestExpectationMode {
     Pass,
     Fail,
+    Skip,
 }
 
 #[derive(Debug, serde::Serialize, serde::Deserialize)]