Merge pull request #50 from AleoHQ/refactor/compiler

Introduces LeoParser, removes pest for leo-compiler

Cargo.lock

@@ -399,15 +399,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
-[[package]]
-name = "heck"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "hermit-abi"
 version = "0.1.8"
@@ -492,7 +483,6 @@ dependencies = [
  "snarkos-gadgets",
  "snarkos-models",
  "snarkos-utilities",
- "structopt",
  "thiserror",
  "toml",
 ]
@@ -514,12 +504,10 @@ dependencies = [
 name = "leo-compiler"
 version = "0.1.0"
 dependencies = [
- "from-pest",
  "hex",
  "leo-ast",
  "leo-types",
  "log",
- "pest",
  "rand",
  "sha2",
  "snarkos-algorithms",
@@ -736,32 +724,6 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
 
-[[package]]
-name = "proc-macro-error"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678"
-dependencies = [
- "proc-macro-error-attr",
- "proc-macro2 1.0.9",
- "quote 1.0.3",
- "syn 1.0.16",
- "version_check",
-]
-
-[[package]]
-name = "proc-macro-error-attr"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53"
-dependencies = [
- "proc-macro2 1.0.9",
- "quote 1.0.3",
- "syn 1.0.16",
- "syn-mid",
- "version_check",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "0.4.30"
@@ -1025,7 +987,6 @@ checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4"
 [[package]]
 name = "snarkos-algorithms"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 dependencies = [
  "blake2",
  "derivative",
@@ -1044,7 +1005,6 @@ dependencies = [
 [[package]]
 name = "snarkos-curves"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 dependencies = [
  "derivative",
  "rand",
@@ -1056,7 +1016,6 @@ dependencies = [
 [[package]]
 name = "snarkos-errors"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 dependencies = [
  "bincode",
  "curl",
@@ -1069,7 +1028,6 @@ dependencies = [
 [[package]]
 name = "snarkos-gadgets"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 dependencies = [
  "derivative",
  "digest",
@@ -1083,7 +1041,6 @@ dependencies = [
 [[package]]
 name = "snarkos-models"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 dependencies = [
  "derivative",
  "rand",
@@ -1096,12 +1053,10 @@ dependencies = [
 [[package]]
 name = "snarkos-profiler"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 
 [[package]]
 name = "snarkos-utilities"
 version = "0.8.0"
-source = "git+ssh://git@github.com/AleoHQ/snarkOS.git#ffef82772617e849badf05fbee283d9f34812d31"
 dependencies = [
  "rand",
 ]
@@ -1124,30 +1079,6 @@ version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
 
-[[package]]
-name = "structopt"
-version = "0.3.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef"
-dependencies = [
- "clap",
- "lazy_static",
- "structopt-derive",
-]
-
-[[package]]
-name = "structopt-derive"
-version = "0.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a"
-dependencies = [
- "heck",
- "proc-macro-error",
- "proc-macro2 1.0.9",
- "quote 1.0.3",
- "syn 1.0.16",
-]
-
 [[package]]
 name = "subtle"
 version = "1.0.0"
@@ -1176,17 +1107,6 @@ dependencies = [
  "unicode-xid 0.2.0",
 ]
 
-[[package]]
-name = "syn-mid"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
-dependencies = [
- "proc-macro2 1.0.9",
- "quote 1.0.3",
- "syn 1.0.16",
-]
-
 [[package]]
 name = "synstructure"
 version = "0.12.3"
@@ -1267,12 +1187,6 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f00ed7be0c1ff1e24f46c3d2af4859f7e863672ba3a6e92e7cff702bf9f06c2"
 
-[[package]]
-name = "unicode-segmentation"
-version = "1.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
-
 [[package]]
 name = "unicode-width"
 version = "0.1.7"

Cargo.toml

@@ -18,12 +18,12 @@ members = [ "ast", "compiler", "types" ]
 [dependencies]
 leo-compiler = { path = "compiler", version = "0.1.0" }
 
-snarkos-algorithms = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-algorithms = { path = "../snarkOS/algorithms", version = "0.8.0" }
-snarkos-curves = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-curves = { path = "../snarkOS/curves", version = "0.8.0" }
-snarkos-errors = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-errors = { path = "../snarkOS/errors", version = "0.8.0" }
-snarkos-gadgets = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-gadgets = { path = "../snarkOS/gadgets", version = "0.8.0" }
-snarkos-models =  { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-models =  { path = "../snarkOS/models", version = "0.8.0" }
-snarkos-utilities = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-utilities = { path = "../snarkOS/utilities", version = "0.8.0" }
 
 clap = { version = "2.33.0" }
 colored = { version = "1.9" }
@@ -34,6 +34,5 @@ rand = { version = "0.7" }
 rand_core = { version = "0.5.1" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "1.0" }
-structopt = { version = "0.3.14" }
 toml = { version = "0.5" }
 thiserror = { version = "1.0" }

ast/src/errors/mod.rs

@@ -1,2 +1,5 @@
+pub mod parser;
+pub use parser::*;
+
 pub mod syntax;
 pub use syntax::*;

ast/src/errors/parser.rs

@@ -0,0 +1,22 @@
+use crate::{ast::Rule, errors::SyntaxError};
+
+use pest::error::Error;
+use std::path::PathBuf;
+
+#[derive(Debug, Error)]
+pub enum ParserError {
+    #[error("Cannot read from the provided file path - {:?}", _0)]
+    FileReadError(PathBuf),
+
+    #[error("{}", _0)]
+    SyntaxError(#[from] SyntaxError),
+
+    #[error("Unable to construct abstract syntax tree")]
+    SyntaxTreeError,
+}
+
+impl From<Error<Rule>> for ParserError {
+    fn from(error: Error<Rule>) -> Self {
+        ParserError::SyntaxError(SyntaxError::from(error))
+    }
+}

ast/src/lib.rs

@@ -9,7 +9,7 @@ pub mod errors;
 pub use errors::*;
 
 pub mod access;
-pub mod ast;
+mod ast;
 pub mod circuits;
 pub mod common;
 pub mod expressions;
@@ -20,3 +20,29 @@ pub mod operations;
 pub mod statements;
 pub mod values;
 pub mod types;
+
+use from_pest::FromPest;
+use std::{path::PathBuf, fs};
+
+pub struct LeoParser;
+
+impl LeoParser {
+    /// Reads in the given file path into a string.
+    pub fn load_file(file_path: &PathBuf) -> Result<String, ParserError> {
+        Ok(fs::read_to_string(file_path).map_err(|_| ParserError::FileReadError(file_path.clone()))?)
+    }
+
+    /// Parses the input file and constructs a syntax tree.
+    pub fn parse_file<'a>(file_path: &'a PathBuf, input_file: &'a str) -> Result<files::File<'a>, ParserError> {
+        // Parse the file using leo.pest
+        let mut file = ast::parse(input_file).map_err(|error| {
+            ParserError::from(error.with_path(file_path.to_str().unwrap()))
+        })?;
+
+        // Build the abstract syntax tree
+        let syntax_tree = files::File::from_pest(&mut file).map_err(|_| ParserError::SyntaxTreeError)?;
+        log::debug!("{:#?}", syntax_tree);
+
+        Ok(syntax_tree)
+    }
+}

compiler/Cargo.toml

@@ -8,17 +8,15 @@ edition = "2018"
 leo-ast = { path = "../ast", version = "0.1.0" }
 leo-types = { path = "../types", version = "0.1.0" }
 
-snarkos-algorithms = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-algorithms = { path = "../../snarkOS/algorithms", version = "0.8.0" }
-snarkos-curves = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-curves = { path = "../../snarkOS/curves", version = "0.8.0" }
-snarkos-errors = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-errors = { path = "../../snarkOS/errors", version = "0.8.0" }
-snarkos-gadgets = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-gadgets = { path = "../../snarkOS/gadgets", version = "0.8.0" }
-snarkos-models = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-models =  { path = "../../snarkOS/models", version = "0.8.0" }
-snarkos-utilities = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-utilities = { path = "../../snarkOS/utilities", version = "0.8.0" }
 
-from-pest = { version = "0.3.1" }
 hex = { version = "0.4.2" }
 log = { version = "0.4" }
-pest = { version = "2.0" }
 rand = { version = "0.7" }
 sha2 = { version = "0.8" }
 thiserror = { version = "1.0" }

compiler/src/compiler.rs

@@ -5,7 +5,7 @@ use crate::{
     errors::CompilerError,
     GroupType,
 };
-use leo_ast::{ast, files::File};
+use leo_ast::LeoParser;
 use leo_types::{InputValue, Program};
 
 use snarkos_errors::gadgets::SynthesisError;
@@ -14,7 +14,6 @@ use snarkos_models::{
     gadgets::r1cs::{ConstraintSynthesizer, ConstraintSystem, TestConstraintSystem},
 };
 
-use from_pest::FromPest;
 use sha2::{Digest, Sha256};
 use std::{fs, marker::PhantomData, path::PathBuf};
 
@@ -76,36 +75,13 @@ impl<F: Field + PrimeField, G: GroupType<F>> Compiler<F, G> {
         generate_test_constraints::<F, G>(cs, self.program)
     }
 
-    // pub fn compile(&self) -> Result<File, CompilerError> {
-    //     // Read in the main file as string
-    //     let unparsed_file = fs::read_to_string(&self.main_file_path).map_err(|_| CompilerError::FileReadError(self.main_file_path.clone()))?;
-    //
-    //     // Parse the file using leo.pest
-    //     let mut file = ast::parse(&unparsed_file).map_err(|_| CompilerError::FileParsingError)?;
-    //
-    //     // Build the abstract syntax tree
-    //     let syntax_tree = File::from_pest(&mut file).map_err(|_| CompilerError::SyntaxTreeError)?;
-    //     log::debug!("{:#?}", syntax_tree);
-    //
-    //     Ok(syntax_tree)
-    // }
-
     fn parse_program(&mut self) -> Result<(), CompilerError> {
-        // Read in the main file as string
+        // Build the program syntax tree
-        let unparsed_file = fs::read_to_string(&self.main_file_path)
+        let file_path = &self.main_file_path;
-            .map_err(|_| CompilerError::FileReadError(self.main_file_path.clone()))?;
+        let input_file = &LeoParser::load_file(file_path)?;
+        let syntax_tree = LeoParser::parse_file(file_path, input_file)?;
 
-        // Parse the file using leo.pest
+        // Build program from syntax tree
-        let mut file = ast::parse(&unparsed_file).map_err(|error| {
-            CompilerError::from(error.with_path(&self.main_file_path.to_str().unwrap()))
-        })?;
-
-        // Build the abstract syntax tree
-        let syntax_tree =
-            File::from_pest(&mut file).map_err(|_| CompilerError::SyntaxTreeError)?;
-        log::debug!("{:#?}", syntax_tree);
-
-        // Build program from abstract syntax tree
         let package_name = self.package_name.clone();
 
         self.program = Program::from(syntax_tree, package_name);

compiler/src/constraints/import.rs

@@ -4,15 +4,14 @@ use crate::{
     new_scope,
     GroupType,
 };
-use leo_ast::{ast, files::File};
+use leo_ast::LeoParser;
 use leo_types::{Import, Program};
 
-use from_pest::FromPest;
 use snarkos_models::{
     curves::{Field, PrimeField},
     gadgets::r1cs::ConstraintSystem,
 };
-use std::{env::current_dir, fs};
+use std::env::current_dir;
 
 impl<F: Field + PrimeField, G: GroupType<F>, CS: ConstraintSystem<F>> ConstrainedProgram<F, G, CS> {
     pub fn enforce_import(
@@ -35,16 +34,12 @@ impl<F: Field + PrimeField, G: GroupType<F>, CS: ConstraintSystem<F>> Constraine
 
         println!("Compiling import - {:?}", main_file_path);
 
-        // Resolve program file path
+        // Build the abstract syntax tree
-        let unparsed_file = fs::read_to_string(main_file_path.clone())
+        let file_path = &main_file_path;
-            .map_err(|_| ImportError::FileReadError(main_file_path))?;
+        let input_file = &LeoParser::load_file(file_path)?;
-        let mut file = ast::parse(&unparsed_file).map_err(|_| ImportError::FileParsingError)?;
+        let syntax_tree = LeoParser::parse_file(file_path, input_file)?;
 
-        // generate ast from file
+        // Generate aleo program from file
-        let syntax_tree =
-            File::from_pest(&mut file).map_err(|_| ImportError::SyntaxTreeError)?;
-
-        // generate aleo program from file
         let mut program = Program::from(syntax_tree, import.path_string.clone());
 
         // Use same namespace as calling function for imported symbols

compiler/src/errors/compiler.rs

@@ -1,8 +1,7 @@
 use crate::errors::{FunctionError, ImportError};
-use leo_ast::{ast::Rule, SyntaxError};
+use leo_ast::{SyntaxError, ParserError};
 use leo_types::IntegerError;
 
-use pest::error::Error;
 use std::{io, path::PathBuf};
 
 #[derive(Debug, Error)]
@@ -34,6 +33,9 @@ pub enum CompilerError {
     #[error("Main must be a function")]
     NoMainFunction,
 
+    #[error("{}", _0)]
+    ParserError(#[from] ParserError),
+
     #[error("{}", _0)]
     SyntaxError(#[from] SyntaxError),
 
@@ -43,9 +45,3 @@ pub enum CompilerError {
     #[error("writing: {}", _0)]
     Writing(io::Error),
 }
-
-impl From<Error<Rule>> for CompilerError {
-    fn from(error: Error<Rule>) -> Self {
-        CompilerError::SyntaxError(SyntaxError::from(error))
-    }
-}

compiler/src/errors/constraints/import.rs

@@ -1,3 +1,5 @@
+use leo_ast::ParserError;
+
 use std::{io, path::PathBuf};
 
 #[derive(Debug, Error)]
@@ -11,6 +13,9 @@ pub enum ImportError {
     #[error("Cannot read from the provided file path - {:?}", _0)]
     FileReadError(PathBuf),
 
+    #[error("{}", _0)]
+    ParserError(#[from] ParserError),
+
     #[error("Unable to construct abstract syntax tree")]
     SyntaxTreeError,
 }

compiler/tests/syntax/mod.rs

@@ -10,7 +10,7 @@ fn test_semicolon() {
         .unwrap();
 
     match error {
-        CompilerError::SyntaxError(_) => {}
+        CompilerError::ParserError(_) => {}
-        _ => panic!("wrong error"),
+        _ => panic!("test_semicolon failed the wrong expected error, should be a ParserError"),
     }
 }

types/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2018"
 [dependencies]
 leo-ast = { path = "../ast", version = "0.1.0" }
 
-snarkos-errors = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-errors = { path = "../../snarkOS/errors", version = "0.8.0" }
-snarkos-models = { git = "ssh://git@github.com/AleoHQ/snarkOS.git", version = "0.8.0" }
+snarkos-models =  { path = "../../snarkOS/models", version = "0.8.0" }
 
 thiserror = { version = "1.0" }

types/src/expression.rs

@@ -277,7 +277,6 @@ impl<'ast> From<Assignee<'ast>> for Expression {
     }
 }
 
-
 impl<'ast> From<BinaryExpression<'ast>> for Expression {
     fn from(expression: BinaryExpression<'ast>) -> Self {
         match expression.operation {