From 8972674dec783a28dc8722012da7e3122860db1a Mon Sep 17 00:00:00 2001 From: gluax <16431709+gluax@users.noreply.github.com> Date: Wed, 8 Jun 2022 13:44:40 -0700 Subject: [PATCH] test-framework rework and clean up start --- tests/test-framework/Cargo.toml | 57 ++- tests/test-framework/benches/leo_compiler.rs | 193 +++++++++++ tests/test-framework/src/fetch.rs | 34 +- tests/test-framework/src/runner.rs | 344 +++++++++++-------- 4 files changed, 429 insertions(+), 199 deletions(-) create mode 100644 tests/test-framework/benches/leo_compiler.rs diff --git a/tests/test-framework/Cargo.toml b/tests/test-framework/Cargo.toml index ff592d09c4..b4742819b3 100644 --- a/tests/test-framework/Cargo.toml +++ b/tests/test-framework/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "leo-test-framework" version = "1.5.3" -authors = [ "The Aleo Team " ] +authors = ["The Aleo Team "] description = "Leo testing framework" homepage = "https://aleo.org" repository = "https://github.com/AleoHQ/leo" @@ -10,61 +10,50 @@ keywords = [ "cryptography", "leo", "programming-language", - "zero-knowledge" + "zero-knowledge", ] -categories = [ "cryptography::cryptocurrencies", "web-programming" ] -include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ] +categories = ["cryptography::cryptocurrencies", "web-programming"] +include = ["Cargo.toml", "src", "README.md", "LICENSE.md"] license = "GPL-3.0" edition = "2018" +[[bench]] +name = "leo_compiler" +path = "benches/leo_compiler.rs" +harness = false + [dependencies] backtrace = "0.3.65" +walkdir = "2.3.2" [dependencies.serde] version = "1.0" -features = [ "derive" ] +features = ["derive"] [dependencies.serde_json] version = "1.0" -features = [ "preserve_order" ] +features = ["preserve_order"] [dependencies.serde_yaml] version = "0.8" -# List of dependencies for tgc binary; -# disabled for now while porting modules from staging -# [dependencies.leo-ast] -# path = "../ast" -# version = "1.5.2" - -# [dependencies.leo-ast-passes] -# path = "../ast-passes" -# version = "1.5.2" - -# [dependencies.leo-parser] -# path = "../parser" -# version = "1.5.2" - -# [dependencies.leo-imports] -# path = "../imports" -# version = "1.5.2" - -# [dependencies.leo-asg] -# path = "../asg" -# version = "1.5.2" - -# [dependencies.leo-compiler] -# path = "../compiler" -# version = "1.5.2" - [dependencies.structopt] version = "0.3" -# List of dependencies for errcov - [dependencies.leo-errors] path = "../../leo/errors" version = "1.5.3" [dependencies.regex] version = "1.5" + +[dev-dependencies.criterion] +version = "0.3" + +[dev-dependencies.leo-compiler] +path = "../../compiler/compiler" +version = "1.5.3" + +[dev-dependencies.leo-span] +path = "../../leo/span" +version = "1.5.3" diff --git a/tests/test-framework/benches/leo_compiler.rs b/tests/test-framework/benches/leo_compiler.rs new file mode 100644 index 0000000000..935a8d53ab --- /dev/null +++ b/tests/test-framework/benches/leo_compiler.rs @@ -0,0 +1,193 @@ +// Copyright (C) 2019-2022 Aleo Systems Inc. +// This file is part of the Leo library. + +// The Leo library is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// The Leo library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with the Leo library. If not, see . + +use leo_compiler::Compiler; +use leo_errors::emitter::Handler; +use leo_span::{ + source_map::FileName, + symbol::{SessionGlobals, SESSION_GLOBALS}, +}; +use leo_test_framework::{ + runner::{Namespace, ParseType, Runner}, + Test, +}; + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; + +macro_rules! sample { + ($name:expr) => { + Sample { + name: $name, + input: include_str!(concat!("./", $name, ".leo")), + path: concat!("./", $name, ".leo"), + } + }; +} + +#[derive(Clone, Copy)] +struct Sample { + name: &'static str, + input: &'static str, + path: &'static str, +} + +fn new_compiler<'a>(handler: &'a Handler, main_file_path: &str) -> Compiler<'a> { + Compiler::new( + handler, + PathBuf::from(main_file_path), + PathBuf::from("/tmp/output/"), + None, + ) +} + +impl Sample { + const SAMPLES: &'static [Sample] = &[sample!("big"), sample!("iteration")]; + + fn data(&self) -> (&str, FileName) { + black_box((self.input, FileName::Custom(self.path.into()))) + } + + fn bench_parse(&self, c: &mut Criterion) { + c.bench_function(&format!("parse {}", self.name), |b| { + b.iter_custom(|iters| { + let mut time = Duration::default(); + for _ in 0..iters { + SESSION_GLOBALS.set(&SessionGlobals::default(), || { + let handler = Handler::default(); + let mut compiler = new_compiler(&handler, self.path); + let (input, name) = self.data(); + let start = Instant::now(); + let out = compiler.parse_program_from_string(input, name); + time += start.elapsed(); + out.expect("Failed to parse program") + }); + } + time + }) + }); + } + + fn bench_symbol_table(&self, c: &mut Criterion) { + c.bench_function(&format!("symbol table pass {}", self.name), |b| { + b.iter_custom(|iters| { + let mut time = Duration::default(); + for _ in 0..iters { + SESSION_GLOBALS.set(&SessionGlobals::default(), || { + let handler = Handler::default(); + let mut compiler = new_compiler(&handler, self.path); + let (input, name) = self.data(); + compiler + .parse_program_from_string(input, name) + .expect("Failed to parse program"); + let start = Instant::now(); + let out = compiler.symbol_table_pass(); + time += start.elapsed(); + out.expect("failed to generate symbol table"); + }); + } + time + }) + }); + } + + fn bench_type_checker(&self, c: &mut Criterion) { + c.bench_function(&format!("type checker pass {}", self.name), |b| { + b.iter_custom(|iters| { + let mut time = Duration::default(); + for _ in 0..iters { + SESSION_GLOBALS.set(&SessionGlobals::default(), || { + let handler = Handler::default(); + let mut compiler = new_compiler(&handler, self.path); + let (input, name) = self.data(); + compiler + .parse_program_from_string(input, name) + .expect("Failed to parse program"); + let mut symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table"); + let start = Instant::now(); + let out = compiler.type_checker_pass(&mut symbol_table); + time += start.elapsed(); + out.expect("failed to run type check pass") + }); + } + time + }) + }); + } + + fn bench_full(&self, c: &mut Criterion) { + c.bench_function(&format!("full {}", self.name), |b| { + b.iter_custom(|iters| { + let mut time = Duration::default(); + for _ in 0..iters { + SESSION_GLOBALS.set(&SessionGlobals::default(), || { + let handler = Handler::default(); + let mut compiler = new_compiler(&handler, self.path); + let (input, name) = self.data(); + let start = Instant::now(); + compiler + .parse_program_from_string(input, name) + .expect("Failed to parse program"); + let mut symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table"); + compiler + .type_checker_pass(&mut symbol_table) + .expect("failed to run type check pass"); + time += start.elapsed(); + }); + } + time + }) + }); + } +} + +fn bench_parse(c: &mut Criterion) { + for sample in Sample::SAMPLES { + sample.bench_parse(c); + } +} + +fn bench_symbol_table(c: &mut Criterion) { + for sample in Sample::SAMPLES { + sample.bench_symbol_table(c); + } +} + +fn bench_type_checker(c: &mut Criterion) { + for sample in Sample::SAMPLES { + sample.bench_type_checker(c); + } +} + +fn bench_full(c: &mut Criterion) { + for sample in Sample::SAMPLES { + sample.bench_full(c); + } +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(200).measurement_time(Duration::from_secs(30)).nresamples(200_000); + targets = + bench_parse, + bench_symbol_table, + bench_type_checker, + bench_full +); +criterion_main!(benches); diff --git a/tests/test-framework/src/fetch.rs b/tests/test-framework/src/fetch.rs index 862df5035b..4bb1b1b502 100644 --- a/tests/test-framework/src/fetch.rs +++ b/tests/test-framework/src/fetch.rs @@ -16,25 +16,31 @@ use std::{fs, path::Path}; -pub fn find_tests>(path: T, out: &mut Vec<(String, String)>) { - for entry in fs::read_dir(path).expect("fail to read tests") { - let entry = entry.expect("fail to read tests").path(); - if entry.is_dir() { - find_tests(entry.as_path(), out); - continue; - } else if entry.extension().and_then(|x| x.to_str()).unwrap_or_default() != "leo" { - continue; - } - let content = fs::read_to_string(entry.as_path()).expect("failed to read test"); - out.push((entry.as_path().to_str().unwrap_or_default().to_string(), content)); - } +use walkdir::WalkDir; + +pub fn find_tests + Copy>(path: T, filter: T) -> Vec<(String, String)> { + let count = WalkDir::new(path) + .into_iter() + .flatten() + .filter_map(|f| { + let path = f.path(); + if matches!(path.extension(), Some(s) if s == "leo") && !path.starts_with(filter) { + let content = fs::read_to_string(path).expect("failed to read test"); + Some((path.to_str().unwrap_or_default().to_string(), content)) + } else { + None + } + }) + .collect::>(); + dbg!("find_tests count {}", count.len()); + count } -pub fn split_tests_oneline(source: &str) -> Vec<&str> { +pub fn split_tests_one_line(source: &str) -> Vec<&str> { source.lines().map(|x| x.trim()).filter(|x| !x.is_empty()).collect() } -pub fn split_tests_twoline(source: &str) -> Vec { +pub fn split_tests_two_line(source: &str) -> Vec { let mut out = vec![]; let mut lines = vec![]; for line in source.lines() { diff --git a/tests/test-framework/src/runner.rs b/tests/test-framework/src/runner.rs index 8beb27fbe8..486b7bf920 100644 --- a/tests/test-framework/src/runner.rs +++ b/tests/test-framework/src/runner.rs @@ -81,51 +81,96 @@ fn take_hook( output.map_err(|_| panic_buf.lock().unwrap().take().expect("failed to get panic message")) } -pub fn run_tests(runner: &T, expectation_category: &str) { - std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files - std::env::set_var("LEO_TESTFRAMEWORK", "true"); - let mut pass_categories = 0; - let mut pass_tests = 0; - let mut fail_tests = 0; - let mut fail_categories = Vec::new(); +pub struct TestCases { + tests: Vec<(String, String)>, + path_prefix: PathBuf, + fail_categories: Vec, +} - let mut tests = Vec::new(); - let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - test_dir.push("../../tests/"); +impl TestCases { + fn new(additional_check: impl Fn(&TestConfig) -> bool) -> (Self, Vec) { + let mut path_prefix = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path_prefix.push("../../tests/"); - let mut expectation_dir = test_dir.clone(); - expectation_dir.push("expectations"); + let mut expectation_dir = path_prefix.clone(); + expectation_dir.push("expectations"); - find_tests(&test_dir, &mut tests); + let mut new = Self { + tests: Vec::new(), + path_prefix, + fail_categories: Vec::new(), + }; + let tests = new.load_tests(additional_check); + (new, tests) + } - let filter = std::env::var("TEST_FILTER").unwrap_or_default(); - let filter = filter.trim(); + fn load_tests(&mut self, additional_check: impl Fn(&TestConfig) -> bool) -> Vec { + let filter = PathBuf::from(std::env::var("TEST_FILTER").unwrap_or_default().trim()); + let mut configs = Vec::new(); - let mut outputs = vec![]; + self.tests = find_tests(&self.path_prefix, &filter) + .into_iter() + .filter(|(path, content)| { + let config = match extract_test_config(content) { + None => { + self.fail_categories.push(TestFailure { + path: path.to_string(), + errors: vec![TestError::MissingTestConfig], + }); + return true; + } + Some(cfg) => cfg, + }; - for (path, content) in tests.into_iter() { - if !filter.is_empty() && !path.contains(filter) { - continue; + let res = additional_check(&config); + configs.push(config); + res + }) + .collect(); + + configs + } + + pub(crate) fn process_tests( + &mut self, + configs: Vec, + expectation_category: &str, + mut process: P, + ) -> Vec + where + P: FnMut(&mut Self, (&Path, &str, &str, TestConfig)) -> O, + { + std::env::remove_var("LEO_BACKTRACE"); // always remove backtrace so it doesn't clog output files + std::env::set_var("LEO_TESTFRAMEWORK", "true"); + + let mut output = Vec::new(); + dbg!("in proccesing: tests {}, configs {}", self.tests.len(), configs.len()); + for ((path, content), config) in self.tests.clone().iter().zip(configs.into_iter()) { + let path = Path::new(&path); + let relative_path = path.strip_prefix(&self.path_prefix).expect("path error for test"); + + let test_name = relative_path + .file_stem() + .expect("no file name for test") + .to_str() + .unwrap() + .to_string(); + + let end_of_header = content.find("*/").expect("failed to find header block in test"); + let content = &content[end_of_header + 2..]; + + output.push(process(self, (path, content, &test_name, config))); + + std::env::remove_var("LEO_TESTFRAMEWORK"); } - let config = extract_test_config(&content); - if config.is_none() { - //panic!("missing configuration for {}", path); - // fail_categories.push(TestFailure { - // path, - // errors: vec![TestError::MissingTestConfig], - // }); - continue; - } - let config = config.unwrap(); - let namespace = runner.resolve_namespace(&config.namespace); - if namespace.is_none() { - continue; - } - let namespace = namespace.unwrap(); + output + } + fn clear_expectations(&self, path: &Path, expectation_category: &str) -> (PathBuf, Option) { let path = Path::new(&path); - let relative_path = path.strip_prefix(&test_dir).expect("path error for test"); - let mut expectation_path = expectation_dir.clone(); + let relative_path = path.strip_prefix(&self.path_prefix).expect("path error for test"); + let expectation_dir = self.path_prefix.clone(); + let mut expectation_path = expectation_dir; expectation_path.push(expectation_category); expectation_path.push(relative_path.parent().expect("no parent dir for test")); let mut expectation_name = relative_path @@ -137,132 +182,129 @@ pub fn run_tests(runner: &T, expectation_category: &str) { expectation_name += ".out"; expectation_path.push(&expectation_name); - let test_name = relative_path - .file_stem() - .expect("no file name for test") - .to_str() - .unwrap() - .to_string(); - - let expectations: Option = if expectation_path.exists() { + if expectation_path.exists() { if !std::env::var("CLEAR_LEO_TEST_EXPECTATIONS") .unwrap_or_default() .trim() .is_empty() { - None + (expectation_path, None) } else { let raw = std::fs::read_to_string(&expectation_path).expect("failed to read expectations file"); - Some(serde_yaml::from_str(&raw).expect("invalid yaml in expectations file")) - } - } else { - None - }; - - let end_of_header = content.find("*/").expect("failed to find header block in test"); - let content = &content[end_of_header + 2..]; - - let tests = match namespace.parse_type() { - ParseType::Line => crate::fetch::split_tests_oneline(content) - .into_iter() - .map(|x| x.to_string()) - .collect(), - ParseType::ContinuousLines => crate::fetch::split_tests_twoline(content), - ParseType::Whole => vec![content.to_string()], - }; - - let mut errors = vec![]; - if let Some(expectations) = expectations.as_ref() { - if tests.len() != expectations.outputs.len() { - errors.push(TestError::MismatchedTestExpectationLength); - } - } - - let mut new_outputs = vec![]; - - let mut expected_output = expectations.as_ref().map(|x| x.outputs.iter()); - for (i, test) in tests.into_iter().enumerate() { - let expected_output = expected_output.as_mut().and_then(|x| x.next()).cloned(); - println!("running test {} @ '{}'", test_name, path.to_str().unwrap()); - let panic_buf = set_hook(); - let leo_output = panic::catch_unwind(|| { - namespace.run_test(Test { - name: test_name.clone(), - content: test.clone(), - path: path.into(), - config: config.extra.clone(), - }) - }); - let output = take_hook(leo_output, panic_buf); - if let Some(error) = emit_errors(&test, &output, &config.expectation, expected_output, i) { - fail_tests += 1; - errors.push(error); - } else { - pass_tests += 1; - new_outputs.push( - output - .unwrap() - .as_ref() - .map(|x| serde_yaml::to_value(x).expect("serialization failed")) - .unwrap_or_else(|e| Value::String(e.clone())), - ); - } - } - - if errors.is_empty() { - if expectations.is_none() { - outputs.push(( + ( expectation_path, - TestExpectation { - namespace: config.namespace, - expectation: config.expectation, - outputs: new_outputs, - }, - )); + Some(serde_yaml::from_str(&raw).expect("invalid yaml in expectations file")), + ) } - pass_categories += 1; } else { - fail_categories.push(TestFailure { - path: path.to_str().unwrap().to_string(), - errors, - }) + (expectation_path, None) } } - if !fail_categories.is_empty() { - for (i, fail) in fail_categories.iter().enumerate() { - println!( - "\n\n-----------------TEST #{} FAILED (and shouldn't have)-----------------", - i + 1 - ); - println!("File: {}", fail.path); - for error in &fail.errors { - println!("{}", error); - } - } - panic!( - "failed {}/{} tests in {}/{} categories", - pass_tests, - fail_tests + pass_tests, - fail_categories.len(), - fail_categories.len() + pass_categories - ); - } else { - for (path, new_expectation) in outputs { - std::fs::create_dir_all(path.parent().unwrap()).expect("failed to make test expectation parent directory"); - std::fs::write( - &path, - serde_yaml::to_string(&new_expectation).expect("failed to serialize expectation yaml"), - ) - .expect("failed to write expectation file"); - } - println!( - "passed {}/{} tests in {}/{} categories", - pass_tests, - fail_tests + pass_tests, - pass_categories, - pass_categories - ); - } - std::env::remove_var("LEO_TESTFRAMEWORK"); + pub fn run_tests(runner: &T, expectation_category: &str) { + let (mut cases, configs) = Self::new(|_| false); + + let mut pass_categories = 0; + let mut pass_tests = 0; + let mut fail_tests = 0; + + let mut outputs = vec![]; + dbg!("running tests"); + cases.process_tests( + configs, + expectation_category, + |cases, (path, content, test_name, config)| { + dbg!("processing tests"); + let namespace = match runner.resolve_namespace(&config.namespace) { + None => todo!("continue"), + Some(ns) => ns, + }; + + let (expectation_path, expectations) = cases.clear_expectations(path, expectation_category); + + let tests = match namespace.parse_type() { + ParseType::Line => crate::fetch::split_tests_one_line(content) + .into_iter() + .map(|x| x.to_string()) + .collect(), + ParseType::ContinuousLines => crate::fetch::split_tests_two_line(content), + ParseType::Whole => vec![content.to_string()], + }; + dbg!("tests len{}", tests.len()); + + let mut errors = vec![]; + if let Some(expectations) = expectations.as_ref() { + if tests.len() != expectations.outputs.len() { + errors.push(TestError::MismatchedTestExpectationLength); + } + } + + let mut new_outputs = vec![]; + let mut expected_output = expectations.as_ref().map(|x| x.outputs.iter()); + for (i, test) in tests.into_iter().enumerate() { + let expected_output = expected_output.as_mut().and_then(|x| x.next()).cloned(); + println!("running test {} @ '{}'", test_name, path.to_str().unwrap()); + let panic_buf = set_hook(); + let leo_output = panic::catch_unwind(|| { + namespace.run_test(Test { + name: test_name.to_string(), + content: test.clone(), + path: path.into(), + config: config.extra.clone(), + }) + }); + let output = take_hook(leo_output, panic_buf); + if let Some(error) = emit_errors(&test, &output, &config.expectation, expected_output, i) { + fail_tests += 1; + errors.push(error); + } else { + pass_tests += 1; + new_outputs.push( + output + .unwrap() + .as_ref() + .map(|x| serde_yaml::to_value(x).expect("serialization failed")) + .unwrap_or_else(|e| Value::String(e.clone())), + ); + } + } + + if errors.is_empty() { + if expectations.is_none() { + outputs.push(( + expectation_path, + TestExpectation { + namespace: config.namespace, + expectation: config.expectation, + outputs: new_outputs, + }, + )); + } + pass_categories += 1; + } else { + cases.fail_categories.push(TestFailure { + path: path.to_str().unwrap().to_string(), + errors, + }) + } + + todo!() + }, + ); + } +} + +struct Bencher; + +impl Bencher { + fn get_benches() -> Vec<(String, String)> { + let (mut cases, configs) = TestCases::new(|config| config.expectation == TestExpectationMode::Fail); + + let expectation_category = "compiler"; + let tests = cases.process_tests(configs, expectation_category, |_, (_, content, test_name, _)| { + (test_name.to_string(), content.to_string()) + }); + + tests + } }