// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see .
//! This file contains tools for benchmarking the Leo compiler and its stages.
use leo_compiler::Compiler;
use leo_errors::emitter::{Emitter, Handler};
use leo_span::{
source_map::FileName,
symbol::{SessionGlobals, SESSION_GLOBALS},
};
use leo_test_framework::get_benches;
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use std::{
path::PathBuf,
time::{Duration, Instant},
};
/// An enum to represent the stage of the Compiler we are benchmarking.
enum BenchMode {
/// Benchmarks parsing.
Parse,
/// Benchmarks symbol table generation.
Symbol,
/// Benchmarks type checking.
Type,
/// Benchmarks loop unrolling.
Unroll,
/// Benchmarks static single assignment.
Ssa,
/// Benchmarks all the above stages.
Full,
}
/// A dummy buffer emitter since we only test on valid programs.
struct BufEmitter;
impl Emitter for BufEmitter {
fn emit_err(&mut self, _: leo_errors::LeoError) {}
fn last_emitted_err_code(&self) -> Option {
None
}
fn emit_warning(&mut self, _: leo_errors::LeoWarning) {}
}
impl BufEmitter {
fn new_handler() -> Handler {
Handler::new(Box::new(Self))
}
}
/// The name of the test, and the test content.
#[derive(Clone)]
struct Sample {
name: String,
input: String,
}
/// A helper function to help create a Leo Compiler struct.
fn new_compiler(handler: &Handler) -> Compiler<'_> {
Compiler::new(
String::new(),
String::new(),
handler,
PathBuf::from(String::new()),
PathBuf::from(String::new()),
None,
)
}
impl Sample {
/// Loads all the benchmark samples.
/// Leverages the test-framework to grab all tests
/// that are passing compiler tests or marked as benchmark tests.
fn load_samples() -> Vec {
get_benches()
.into_iter()
.map(|(name, input)| Self { name, input })
.collect()
}
fn data(&self) -> (&str, FileName) {
black_box((&self.input, FileName::Custom(String::new())))
}
fn bench(&self, c: &mut Criterion, mode: BenchMode) {
match mode {
BenchMode::Parse => self.bench_parse(c),
BenchMode::Symbol => self.bench_symbol_table(c),
BenchMode::Type => self.bench_type_checker(c),
BenchMode::Unroll => self.bench_loop_unroller(c),
BenchMode::Ssa => self.bench_ssa(c),
BenchMode::Full => self.bench_full(c),
}
}
fn bench_parse(&self, c: &mut Criterion) {
c.bench_function(&format!("parse {}", self.name), |b| {
// Iter custom is used so we can use custom timings around the compiler stages.
// This way we can only time the necessary stage.
b.iter_custom(|iters| {
let mut time = Duration::default();
for _ in 0..iters {
SESSION_GLOBALS.set(&SessionGlobals::default(), || {
let handler = BufEmitter::new_handler();
let mut compiler = new_compiler(&handler);
let (input, name) = self.data();
let start = Instant::now();
let out = compiler.parse_program_from_string(input, name);
time += start.elapsed();
out.expect("Failed to parse program")
});
}
time
})
});
}
fn bench_symbol_table(&self, c: &mut Criterion) {
c.bench_function(&format!("symbol table pass {}", self.name), |b| {
// Iter custom is used so we can use custom timings around the compiler stages.
// This way we can only time the necessary stage.
b.iter_custom(|iters| {
let mut time = Duration::default();
for _ in 0..iters {
SESSION_GLOBALS.set(&SessionGlobals::default(), || {
let handler = BufEmitter::new_handler();
let mut compiler = new_compiler(&handler);
let (input, name) = self.data();
compiler
.parse_program_from_string(input, name)
.expect("Failed to parse program");
let start = Instant::now();
let out = compiler.symbol_table_pass();
time += start.elapsed();
out.expect("failed to generate symbol table");
});
}
time
})
});
}
fn bench_type_checker(&self, c: &mut Criterion) {
c.bench_function(&format!("type checker pass {}", self.name), |b| {
// Iter custom is used so we can use custom timings around the compiler stages.
// This way we can only time the necessary stage.
b.iter_custom(|iters| {
let mut time = Duration::default();
for _ in 0..iters {
SESSION_GLOBALS.set(&SessionGlobals::default(), || {
let handler = BufEmitter::new_handler();
let mut compiler = new_compiler(&handler);
let (input, name) = self.data();
compiler
.parse_program_from_string(input, name)
.expect("Failed to parse program");
let symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table");
let start = Instant::now();
let out = compiler.type_checker_pass(symbol_table);
time += start.elapsed();
out.expect("failed to run type check pass")
});
}
time
})
});
}
fn bench_loop_unroller(&self, c: &mut Criterion) {
c.bench_function(&format!("loop unrolling pass{}", self.name), |b| {
// Iter custom is used so we can use custom timings around the compiler stages.
// This way we can only time the necessary stage.
b.iter_custom(|iters| {
let mut time = Duration::default();
for _ in 0..iters {
SESSION_GLOBALS.set(&SessionGlobals::default(), || {
let handler = BufEmitter::new_handler();
let mut compiler = new_compiler(&handler);
let (input, name) = self.data();
compiler
.parse_program_from_string(input, name)
.expect("Failed to parse program");
let symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table");
let symbol_table = compiler
.type_checker_pass(symbol_table)
.expect("failed to run type check pass");
let start = Instant::now();
let out = compiler.loop_unrolling_pass(symbol_table);
time += start.elapsed();
out.expect("failed to run loop unrolling pass")
});
}
time
})
});
}
fn bench_ssa(&self, c: &mut Criterion) {
c.bench_function(&format!("full {}", self.name), |b| {
// Iter custom is used so we can use custom timings around the compiler stages.
// This way we can only time the necessary stages.
b.iter_custom(|iters| {
let mut time = Duration::default();
for _ in 0..iters {
SESSION_GLOBALS.set(&SessionGlobals::default(), || {
let handler = BufEmitter::new_handler();
let mut compiler = new_compiler(&handler);
let (input, name) = self.data();
compiler
.parse_program_from_string(input, name)
.expect("Failed to parse program");
let symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table");
let symbol_table = compiler
.type_checker_pass(symbol_table)
.expect("failed to run type check pass");
compiler
.loop_unrolling_pass(symbol_table)
.expect("failed to run loop unrolling pass");
let start = Instant::now();
let out = compiler.static_single_assignment_pass();
time += start.elapsed();
out.expect("failed to run ssa pass")
});
}
time
})
});
}
fn bench_full(&self, c: &mut Criterion) {
c.bench_function(&format!("full {}", self.name), |b| {
// Iter custom is used so we can use custom timings around the compiler stages.
// This way we can only time the necessary stages.
b.iter_custom(|iters| {
let mut time = Duration::default();
for _ in 0..iters {
SESSION_GLOBALS.set(&SessionGlobals::default(), || {
let handler = BufEmitter::new_handler();
let mut compiler = new_compiler(&handler);
let (input, name) = self.data();
let start = Instant::now();
compiler
.parse_program_from_string(input, name)
.expect("Failed to parse program");
let symbol_table = compiler.symbol_table_pass().expect("failed to generate symbol table");
let symbol_table = compiler
.type_checker_pass(symbol_table)
.expect("failed to run type check pass");
compiler
.loop_unrolling_pass(symbol_table)
.expect("failed to run loop unrolling pass");
compiler
.static_single_assignment_pass()
.expect("failed to run ssa pass");
time += start.elapsed();
});
}
time
})
});
}
}
macro_rules! bench {
($name:ident, $mode:expr) => {
fn $name(c: &mut Criterion) {
Sample::load_samples().into_iter().for_each(|s| s.bench(c, $mode))
}
};
}
bench!(bench_parse, BenchMode::Parse);
bench!(bench_symbol, BenchMode::Symbol);
bench!(bench_type, BenchMode::Type);
bench!(bench_unroll, BenchMode::Unroll);
bench!(bench_ssa, BenchMode::Ssa);
bench!(bench_full, BenchMode::Full);
criterion_group!(
name = benches;
config = Criterion::default().sample_size(200).measurement_time(Duration::from_secs(10)).nresamples(200_000);
targets =
bench_parse,
bench_symbol,
bench_type,
bench_unroll,
bench_ssa,
bench_full
);
criterion_main!(benches);