Implements caching of builds and setups for reusable work

This commit is contained in:
howardwu 2020-05-02 20:10:22 -07:00
parent 7d5a5b71c3
commit 1350834bcd
19 changed files with 340 additions and 84 deletions

8
Cargo.lock generated
View File

@ -347,6 +347,12 @@ dependencies = [
"libc",
]
[[package]]
name = "hex"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35"
[[package]]
name = "humantime"
version = "1.3.0"
@ -422,12 +428,14 @@ version = "0.1.0"
dependencies = [
"failure",
"from-pest",
"hex",
"lazy_static",
"log",
"pest",
"pest-ast",
"pest_derive",
"rand",
"sha2",
"snarkos-algorithms",
"snarkos-curves",
"snarkos-errors",

View File

@ -13,9 +13,11 @@ snarkos-models = { path = "../../snarkOS/models", version = "0.8.0" }
failure = { version = "0.1.5" }
from-pest = { version = "0.3.1" }
hex = { version = "0.4.2" }
lazy_static = { version = "1.3.0" }
log = { version = "0.4" }
pest = { version = "2.0" }
pest-ast = { version = "0.3.3" }
pest_derive = { version = "2.0" }
rand = { version = "0.7" }
sha2 = { version = "0.8" }

View File

@ -8,6 +8,7 @@ use snarkos_models::{
};
use from_pest::FromPest;
use sha2::{Sha256, Digest};
use std::{fs, marker::PhantomData, path::PathBuf};
#[derive(Clone)]
@ -28,7 +29,33 @@ impl<F: Field + PrimeField> Compiler<F> {
}
}
pub fn evaluate_program<CS: ConstraintSystem<F>>(self, cs: &mut CS) -> Result<ResolvedValue<F>, CompilerError> {
pub fn checksum(&self) -> Result<String, CompilerError> {
// Read in the main file as string
let unparsed_file = fs::read_to_string(&self.main_file_path).map_err(|_| CompilerError::FileReadError(self.main_file_path.clone()))?;
// Hash the file contents
let mut hasher = Sha256::new();
hasher.input(unparsed_file.as_bytes());
let hash = hasher.result();
Ok(hex::encode(hash))
}
// pub fn compile(&self) -> Result<ast::File, CompilerError> {
// // Read in the main file as string
// let unparsed_file = fs::read_to_string(&self.main_file_path).map_err(|_| CompilerError::FileReadError(self.main_file_path.clone()))?;
//
// // Parse the file using leo.pest
// let mut file = ast::parse(&unparsed_file).map_err(|_| CompilerError::FileParsingError)?;
//
// // Build the abstract syntax tree
// let syntax_tree = ast::File::from_pest(&mut file).map_err(|_| CompilerError::SyntaxTreeError)?;
// log::debug!("{:#?}", syntax_tree);
//
// Ok(syntax_tree)
// }
pub fn evaluate_program<CS: ConstraintSystem<F>>(&self, cs: &mut CS) -> Result<ResolvedValue<F>, CompilerError> {
// Read in the main file as string
let unparsed_file = fs::read_to_string(&self.main_file_path).map_err(|_| CompilerError::FileReadError(self.main_file_path.clone()))?;
@ -42,7 +69,7 @@ impl<F: Field + PrimeField> Compiler<F> {
// Build program from abstract syntax tree
let package_name = self.package_name.clone();
let program = Program::<'_, F>::from(syntax_tree).name(package_name);
log::info!("Compilation complete:\n{:#?}", program);
log::debug!("Compilation complete\n{:#?}", program);
Ok(ResolvedProgram::generate_constraints(cs, program))
}

View File

@ -308,7 +308,7 @@ impl<F: Field + PrimeField, CS: ConstraintSystem<F>> ResolvedProgram<F, CS> {
match main.clone() {
ResolvedValue::Function(function) => {
let result = resolved_program.enforce_main_function(cs, program_name, function);
log::info!("{}", result);
log::debug!("{}", result);
result
}
_ => unimplemented!("main must be a function"),

View File

@ -1,11 +1,12 @@
use crate::{cli::*, cli_types::*};
use crate::directories::{source::SOURCE_DIRECTORY_NAME, OutputsDirectory};
use crate::errors::{BuildError, CLIError};
use crate::files::{MainFile, MAIN_FILE_NAME};
use crate::files::{MainFile, MAIN_FILE_NAME, ChecksumFile};
use crate::manifest::Manifest;
use leo_compiler::compiler::Compiler;
use snarkos_curves::bls12_377::Fr;
use snarkos_algorithms::snark::KeypairAssembly;
use snarkos_curves::bls12_377::{Bls12_377, Fr};
use clap::ArgMatches;
use std::convert::TryFrom;
@ -16,7 +17,7 @@ pub struct BuildCommand;
impl CLI for BuildCommand {
type Options = ();
type Output = Compiler<Fr>;
type Output = (Compiler<Fr>, bool);
const NAME: NameType = "build";
const ABOUT: AboutType = "Compile the current package";
@ -59,11 +60,39 @@ impl CLI for BuildCommand {
main_file_path.push(SOURCE_DIRECTORY_NAME);
main_file_path.push(MAIN_FILE_NAME);
log::info!("Compiling program located in {:?}", main_file_path);
// Compute the current program checksum
let program = Compiler::<Fr>::init(package_name.clone(), main_file_path.clone());
let checksum = program.checksum()?;
// Compile from the main file path
let program = Compiler::<Fr>::init(package_name, main_file_path);
// If a checksum file exists, check if it differs from the new checksum
let checksum_file = ChecksumFile::new(&package_name);
let checksum_differs = if checksum_file.exists_at(&package_path) {
let previous_checksum = checksum_file.read_from(&package_path)?;
checksum != previous_checksum
} else {
// By default, the checksum differs if there is no checksum to compare against
true
};
Ok(program)
// If checksum differs, compile the program
if checksum_differs {
// Write the new checksum to the outputs directory
checksum_file.write_to(&path, checksum)?;
// Generate the program on the constraint system and verify correctness
let mut cs = KeypairAssembly::<Bls12_377> {
num_inputs: 0,
num_aux: 0,
num_constraints: 0,
at: vec![],
bt: vec![],
ct: vec![],
};
program.evaluate_program(&mut cs)?;
}
log::info!("Compiled program in {:?}", main_file_path);
Ok((program, checksum_differs))
}
}

View File

@ -40,6 +40,7 @@ impl CLI for ProveCommand {
let path = current_dir()?;
let package_name = Manifest::try_from(&path)?.get_package_name();
// Start the timer
let start = Instant::now();
let rng = &mut thread_rng();
@ -52,6 +53,8 @@ impl CLI for ProveCommand {
program_proof.write(&mut proof)?;
ProofFile::new(&package_name).write_to(&path, &proof)?;
log::info!("Completed program proving");
Ok(program_proof)
}
}

View File

@ -1,6 +1,6 @@
use crate::{cli::*, cli_types::*};
use crate::commands::BuildCommand;
use crate::errors::CLIError;
use crate::errors::{CLIError, VerificationKeyFileError};
use crate::files::{ProvingKeyFile, VerificationKeyFile};
use crate::manifest::Manifest;
use leo_compiler::compiler::Compiler;
@ -42,30 +42,66 @@ impl CLI for SetupCommand {
#[cfg_attr(tarpaulin, skip)]
fn output(options: Self::Options) -> Result<Self::Output, CLIError> {
let program = BuildCommand::output(options)?;
let (program, checksum_differs) = BuildCommand::output(options)?;
// Get the package name
let path = current_dir()?;
let package_name = Manifest::try_from(&path)?.get_package_name();
let start = Instant::now();
// Check if a proving key and verification key already exists
let keys_exist = ProvingKeyFile::new(&package_name).exists_at(&path)
&& VerificationKeyFile::new(&package_name).exists_at(&path);
let rng = &mut thread_rng();
let parameters =
generate_random_parameters::<Bls12_377, _, _>(program.clone(), rng).unwrap();
// If keys do not exist or the checksum differs, run the program setup
if !keys_exist || checksum_differs {
// Start the timer
let start = Instant::now();
// Run the program setup operation
let rng = &mut thread_rng();
let parameters =
generate_random_parameters::<Bls12_377, _, _>(program.clone(), rng).unwrap();
let prepared_verifying_key = prepare_verifying_key::<Bls12_377>(&parameters.vk);
// End the timer
log::info!("Setup completed in {:?} milliseconds", start.elapsed().as_millis());
// TODO (howardwu): Convert parameters to a 'proving key' struct for serialization.
// Write the proving key file to the outputs directory
let mut proving_key = vec![];
parameters.write(&mut proving_key)?;
ProvingKeyFile::new(&package_name).write_to(&path, &proving_key)?;
// Write the proving key file to the outputs directory
let mut verification_key = vec![];
prepared_verifying_key.write(&mut verification_key)?;
VerificationKeyFile::new(&package_name).write_to(&path, &verification_key)?;
}
// Read the proving key file from the outputs directory
let proving_key = ProvingKeyFile::new(&package_name).read_from(&path)?;
let parameters = Parameters::<Bls12_377>::read(proving_key.as_slice(), true)?;
// Read the proving key file from the outputs directory
let prepared_verifying_key = prepare_verifying_key::<Bls12_377>(&parameters.vk);
{
// Load the stored verification key from the outputs directory
let stored_vk = VerificationKeyFile::new(&package_name).read_from(&path)?;
log::info!("Setup completed in {:?} milliseconds", start.elapsed().as_millis());
// Convert the prepared_verifying_key to a buffer
let mut verification_key = vec![];
prepared_verifying_key.write(&mut verification_key)?;
// Write the proving key file to the outputs directory
let mut proving_key = vec![];
parameters.write(&mut proving_key)?;
ProvingKeyFile::new(&package_name).write_to(&path, &proving_key)?;
// Check that the constructed prepared verification key matches the stored verification key
let compare: Vec<(u8, u8)> = verification_key.into_iter().zip(stored_vk.into_iter()).collect();
for (a, b) in compare {
if a != b {
return Err(VerificationKeyFileError::IncorrectVerificationKey.into())
}
}
}
// Write the proving key file to the outputs directory
let mut verification_key = vec![];
prepared_verifying_key.write(&mut verification_key)?;
VerificationKeyFile::new(&package_name).write_to(&path, &verification_key)?;
log::info!("Completed program setup");
Ok((program, parameters, prepared_verifying_key))
}

View File

@ -8,6 +8,9 @@ pub enum CLIError {
#[fail(display = "{}: {}", _0, _1)]
Crate(&'static str, String),
#[fail(display = "{}", _0)]
ChecksumFileError(ChecksumFileError),
#[fail(display = "{}", _0)]
InitError(InitError),
@ -48,6 +51,12 @@ impl From<BuildError> for CLIError {
}
}
impl From<ChecksumFileError> for CLIError {
fn from(error: ChecksumFileError) -> Self {
CLIError::ChecksumFileError(error)
}
}
impl From<InitError> for CLIError {
fn from(error: InitError) -> Self {
CLIError::InitError(error)
@ -114,6 +123,12 @@ impl From<VerificationKeyFileError> for CLIError {
}
}
impl From<leo_compiler::errors::CompilerError> for CLIError {
fn from(error: leo_compiler::errors::CompilerError) -> Self {
CLIError::Crate("leo_compiler", format!("{}", error))
}
}
impl From<serde_json::error::Error> for CLIError {
fn from(error: serde_json::error::Error) -> Self {
CLIError::Crate("serde_json", format!("{}", error))

View File

@ -0,0 +1,23 @@
use std::io;
use std::path::PathBuf;
#[derive(Debug, Fail)]
pub enum ChecksumFileError {
#[fail(display = "{}: {}", _0, _1)]
Crate(&'static str, String),
#[fail(display = "creating: {}", _0)]
Creating(io::Error),
#[fail(display = "Cannot read from the provided file path - {:?}", _0)]
FileReadError(PathBuf),
#[fail(display = "writing: {}", _0)]
Writing(io::Error),
}
impl From<std::io::Error> for ChecksumFileError {
fn from(error: std::io::Error) -> Self {
ChecksumFileError::Crate("std::io", format!("{}", error))
}
}

View File

@ -1,3 +1,6 @@
pub mod checksum;
pub use self::checksum::*;
pub mod main;
pub use self::main::*;

View File

@ -1,4 +1,5 @@
use std::io;
use std::path::PathBuf;
#[derive(Debug, Fail)]
pub enum ProofFileError {
@ -8,6 +9,9 @@ pub enum ProofFileError {
#[fail(display = "creating: {}", _0)]
Creating(io::Error),
#[fail(display = "Cannot read from the provided file path - {:?}", _0)]
FileReadError(PathBuf),
#[fail(display = "writing: {}", _0)]
Writing(io::Error),
}

View File

@ -1,4 +1,5 @@
use std::io;
use std::path::PathBuf;
#[derive(Debug, Fail)]
pub enum ProvingKeyFileError {
@ -8,6 +9,9 @@ pub enum ProvingKeyFileError {
#[fail(display = "creating: {}", _0)]
Creating(io::Error),
#[fail(display = "Cannot read from the provided file path - {:?}", _0)]
FileReadError(PathBuf),
#[fail(display = "writing: {}", _0)]
Writing(io::Error),
}

View File

@ -1,4 +1,5 @@
use std::io;
use std::path::PathBuf;
#[derive(Debug, Fail)]
pub enum VerificationKeyFileError {
@ -8,6 +9,12 @@ pub enum VerificationKeyFileError {
#[fail(display = "creating: {}", _0)]
Creating(io::Error),
#[fail(display = "Cannot read from the provided file path - {:?}", _0)]
FileReadError(PathBuf),
#[fail(display = "Verification key file was corrupted")]
IncorrectVerificationKey,
#[fail(display = "writing: {}", _0)]
Writing(io::Error),
}

59
leo/files/checksum.rs Normal file
View File

@ -0,0 +1,59 @@
//! The build checksum file.
use crate::directories::outputs::OUTPUTS_DIRECTORY_NAME;
use crate::errors::ChecksumFileError;
use serde::Deserialize;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
pub static CHECKSUM_FILE_EXTENSION: &str = ".leo.checksum";
#[derive(Deserialize)]
pub struct ChecksumFile {
pub package_name: String,
}
impl ChecksumFile {
pub fn new(package_name: &str) -> Self {
Self {
package_name: package_name.to_string(),
}
}
pub fn exists_at(&self, path: &PathBuf) -> bool {
let path = self.setup_file_path(path);
path.exists()
}
/// Reads the checksum from the given file path if it exists.
pub fn read_from(&self, path: &PathBuf) -> Result<String, ChecksumFileError> {
let path = self.setup_file_path(path);
Ok(fs::read_to_string(&path).map_err(|_| ChecksumFileError::FileReadError(path.clone()))?)
}
/// Writes the given checksum to a file.
pub fn write_to(&self, path: &PathBuf, checksum: String) -> Result<(), ChecksumFileError> {
let path = self.setup_file_path(path);
let mut file = File::create(&path)?;
file.write_all(checksum.as_bytes())?;
log::info!("Checksum stored to {:?}", path);
Ok(())
}
fn setup_file_path(&self, path: &PathBuf) -> PathBuf {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, CHECKSUM_FILE_EXTENSION)));
}
path
}
}

View File

@ -1,3 +1,6 @@
pub mod checksum;
pub use self::checksum::*;
pub mod main;
pub use self::main::*;

View File

@ -1,10 +1,10 @@
//! The `main.leo` file.
//! The proof file.
use crate::directories::outputs::OUTPUTS_DIRECTORY_NAME;
use crate::errors::ProofFileError;
use serde::Deserialize;
use std::fs::File;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
@ -22,31 +22,39 @@ impl ProofFile {
}
}
pub fn exists_at(self, path: &PathBuf) -> bool {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, PROOF_FILE_EXTENSION)));
}
pub fn exists_at(&self, path: &PathBuf) -> bool {
let path = self.setup_file_path(path);
path.exists()
}
pub fn write_to(self, path: &PathBuf, proof: &[u8]) -> Result<(), ProofFileError> {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, PROOF_FILE_EXTENSION)));
}
/// Reads the proof from the given file path if it exists.
pub fn read_from(&self, path: &PathBuf) -> Result<String, ProofFileError> {
let path = self.setup_file_path(path);
let proof = fs::read_to_string(&path).map_err(|_| ProofFileError::FileReadError(path.clone()))?;
Ok(proof)
}
/// Writes the given proof to a file.
pub fn write_to(&self, path: &PathBuf, proof: &[u8]) -> Result<(), ProofFileError> {
let path = self.setup_file_path(path);
let mut file = File::create(&path)?;
file.write_all(proof)?;
log::info!("Proof stored in {:?}", path);
log::info!("Proof stored to {:?}", path);
Ok(())
}
fn setup_file_path(&self, path: &PathBuf) -> PathBuf {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, PROOF_FILE_EXTENSION)));
}
path
}
}

View File

@ -1,10 +1,10 @@
//! The `main.leo` file.
//! The proving key file.
use crate::directories::outputs::OUTPUTS_DIRECTORY_NAME;
use crate::errors::ProvingKeyFileError;
use serde::Deserialize;
use std::fs::File;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
@ -22,31 +22,38 @@ impl ProvingKeyFile {
}
}
pub fn exists_at(self, path: &PathBuf) -> bool {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, PROVING_KEY_FILE_EXTENSION)));
}
pub fn exists_at(&self, path: &PathBuf) -> bool {
let path = self.setup_file_path(path);
path.exists()
}
pub fn write_to(self, path: &PathBuf, proving_key: &[u8]) -> Result<(), ProvingKeyFileError> {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, PROVING_KEY_FILE_EXTENSION)));
}
/// Reads the proving key from the given file path if it exists.
pub fn read_from(&self, path: &PathBuf) -> Result<Vec<u8>, ProvingKeyFileError> {
let path = self.setup_file_path(path);
Ok(fs::read(&path).map_err(|_| ProvingKeyFileError::FileReadError(path.clone()))?)
}
/// Writes the given proving key to a file.
pub fn write_to(&self, path: &PathBuf, proving_key: &[u8]) -> Result<(), ProvingKeyFileError> {
let path = self.setup_file_path(path);
let mut file = File::create(&path)?;
file.write_all(proving_key)?;
log::info!("Proving key stored in {:?}", path);
log::info!("Proving key stored to {:?}", path);
Ok(())
}
fn setup_file_path(&self, path: &PathBuf) -> PathBuf {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, PROVING_KEY_FILE_EXTENSION)));
}
path
}
}

View File

@ -1,10 +1,10 @@
//! The `main.leo` file.
//! The verification key file.
use crate::directories::outputs::OUTPUTS_DIRECTORY_NAME;
use crate::errors::VerificationKeyFileError;
use serde::Deserialize;
use std::fs::File;
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
@ -22,31 +22,38 @@ impl VerificationKeyFile {
}
}
pub fn exists_at(self, path: &PathBuf) -> bool {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, VERIFICATION_KEY_FILE_EXTENSION)));
}
pub fn exists_at(&self, path: &PathBuf) -> bool {
let path = self.setup_file_path(path);
path.exists()
}
pub fn write_to(self, path: &PathBuf, verification_key: &[u8]) -> Result<(), VerificationKeyFileError> {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, VERIFICATION_KEY_FILE_EXTENSION)));
}
/// Reads the verification key from the given file path if it exists.
pub fn read_from(&self, path: &PathBuf) -> Result<Vec<u8>, VerificationKeyFileError> {
let path = self.setup_file_path(path);
Ok(fs::read(&path).map_err(|_| VerificationKeyFileError::FileReadError(path.clone()))?)
}
/// Writes the given verification key to a file.
pub fn write_to(&self, path: &PathBuf, verification_key: &[u8]) -> Result<(), VerificationKeyFileError> {
let path = self.setup_file_path(path);
let mut file = File::create(&path)?;
file.write_all(verification_key)?;
log::info!("Verification key stored in {:?}", path);
log::info!("Verification key stored to {:?}", path);
Ok(())
}
fn setup_file_path(&self, path: &PathBuf) -> PathBuf {
let mut path = path.to_owned();
if path.is_dir() {
if !path.ends_with(OUTPUTS_DIRECTORY_NAME) {
path.push(PathBuf::from(OUTPUTS_DIRECTORY_NAME));
}
path.push(PathBuf::from(format!("{}{}", self.package_name, VERIFICATION_KEY_FILE_EXTENSION)));
}
path
}
}

View File

@ -14,6 +14,17 @@ fn level_string(level: log::Level) -> colored::ColoredString {
}
}
#[allow(dead_code)]
fn colored_string(level: log::Level, message: &str) -> colored::ColoredString {
match level {
log::Level::Error => message.bold().red(),
log::Level::Warn => message.bold().yellow(),
log::Level::Info => message.bold().blue(),
log::Level::Debug => message.bold().magenta(),
log::Level::Trace => message.bold(),
}
}
/// Initialize logger with custom format and verbosity.
///
/// # Arguments
@ -35,9 +46,9 @@ pub fn init_logger(app_name: &'static str, verbosity: usize) {
writeln!(
buf,
"[{:>5} {:>5}] {}",
"{:>5}{:>5} {}",
level_string(record.level()),
app_name,
colored_string(record.level(), app_name),
record.args().to_string().replace("\n", &padding)
)
})