AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2024-12-19 07:32:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

migrate parser

Browse Source
This commit is contained in:
gluax 2022-01-21 12:32:09 -08:00
parent 2a2df78603
commit 3593529c4d
30 changed files with 921 additions and 4916 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
parser/Cargo.toml
View File

@ -16,7 +16,7 @@ categories = [ "cryptography::cryptocurrencies", "web-programming" ]
include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ]
license = "GPL-3.0"
edition = "2021"
rust-version = "1.56.1"
rust-version = "1.56"
[[bench]]
name = "leo_ast"
@ -31,6 +31,14 @@ version = "1.5.3"
path = "../errors"
version = "1.5.3"
[dependencies.leo-input]
path = "../input"
version = "1.5.1"
[dependencies.leo-span]
path = "../span"
version = "1.5.3"
[dependencies.lazy_static]
version = "1.3.0"

4
parser/README.md
View File

@ -30,7 +30,8 @@ Bolded ones are also keywords.
#### Symbols
- At
- Not
- And
- And (`&&`)
- Ampersand (`&`)
- Or
- Eq
- NotEq
@ -98,7 +99,6 @@ Bolded ones are also keywords.
- **If**
- **In**
- **Let**
- **Mut**
- **Return**
- **Static**
- **String**

40
parser/benches/leo_ast.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -14,55 +14,51 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use leo_ast::Ast;
use leo_errors::emitter::Handler;
use leo_span::symbol::create_session_if_not_set_then;
use criterion::{criterion_group, criterion_main, Criterion};
use std::time::Duration;
fn bench_big_if_else(c: &mut Criterion) {
let program_string = include_str!("./big_if_else.leo");
let ast = leo_parser::parse_ast("./big_if_else.leo", program_string).expect("failed to parse benchmark");
fn parse_ast(path: &str, input: &str) -> Ast {
create_session_if_not_set_then(|_| {
leo_parser::parse_ast(&Handler::default(), path, input).expect("failed to parse benchmark")
})
}
fn bench_big_if_else(c: &mut Criterion) {
let ast = parse_ast("./big_if_else.leo", include_str!("./big_if_else.leo"));
c.bench_function("Ast::big_if_else", |b| b.iter(|| &ast));
}
fn bench_big_ternary(c: &mut Criterion) {
let program_string = include_str!("./big_ternary.leo");
let ast = leo_parser::parse_ast("./big_ternary.leo", program_string).expect("failed to parse benchmark");
let ast = parse_ast("./big_ternary.leo", include_str!("./big_ternary.leo"));
c.bench_function("Ast::big_ternary", |b| b.iter(|| &ast));
}
fn bench_big_circuit(c: &mut Criterion) {
let program_string = include_str!("./big_circuit.leo");
let ast = leo_parser::parse_ast("./big_circuit.leo", program_string).expect("failed to parse benchmark");
let ast = parse_ast("./big_circuit.leo", include_str!("./big_circuit.leo"));
c.bench_function("Ast::big_circuit", |b| b.iter(|| &ast));
}
fn bench_long_expr(c: &mut Criterion) {
let program_string = include_str!("./long_expr.leo");
let ast = leo_parser::parse_ast("./long_expr.leo", program_string).expect("failed to parse benchmark");
let ast = parse_ast("./long_expr.leo", include_str!("./long_expr.leo"));
c.bench_function("Ast::long_expr", |b| b.iter(|| &ast));
}
fn bench_long_array(c: &mut Criterion) {
let program_string = include_str!("./long_array.leo");
let ast = leo_parser::parse_ast("./long_array.leo", program_string).expect("failed to parse benchmark");
let ast = parse_ast("./long_array.leo", include_str!("./long_array.leo"));
c.bench_function("Ast::long_array", |b| b.iter(|| &ast));
}
fn bench_many_foos(c: &mut Criterion) {
let program_string = include_str!("./many_foos.leo");
let ast = leo_parser::parse_ast("./many_foos.leo", program_string).expect("failed to parse benchmark");
let ast = parse_ast("./many_foos.leo", include_str!("./many_foos.leo"));
c.bench_function("Ast::many_foos", |b| b.iter(|| &ast));
}
fn bench_many_assigns(c: &mut Criterion) {
let program_string = include_str!("./many_assigns.leo");
let ast = leo_parser::parse_ast("./many_assigns.leo", program_string).expect("failed to parse benchmark");
let ast = parse_ast("./many_assigns.leo", include_str!("./many_assigns.leo"));
c.bench_function("Ast::many_assigns", |b| b.iter(|| &ast));
}

18
parser/examples/parser.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -15,7 +15,9 @@
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use leo_ast::Ast;
use leo_errors::Result;
use leo_errors::{emitter::Handler, Result};
use leo_span::symbol::create_session_if_not_set_then;
use std::{env, fs, path::Path};
fn to_leo_tree(filepath: &Path) -> Result<String> {
@ -23,12 +25,12 @@ fn to_leo_tree(filepath: &Path) -> Result<String> {
let program_filepath = filepath.to_path_buf();
let program_string = fs::read_to_string(&program_filepath).expect("failed to open input file");
// Parses the Leo file and constructs an ast.
let ast = leo_parser::parse_ast(filepath.to_str().unwrap(), &program_string)?;
let serialized_leo_ast = Ast::to_json_string(&ast).expect("serialization failed");
Ok(serialized_leo_ast)
// Parses the Leo file constructing an ast which is then serialized.
create_session_if_not_set_then(|_| {
let handler = Handler::default();
let ast = leo_parser::parse_ast(&handler, filepath.to_str().unwrap(), &program_string)?;
Ok(Ast::to_json_string(&ast).expect("serialization failed"))
})
}
fn main() -> Result<()> {

71
parser/src/lib.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -23,19 +23,82 @@
#![doc = include_str!("../README.md")]
pub(crate) mod tokenizer;
use leo_input::LeoInputParser;
pub use tokenizer::KEYWORD_TOKENS;
pub(crate) use tokenizer::*;
pub mod parser;
pub use parser::*;
use leo_ast::Ast;
use leo_ast::{Ast, Input};
use leo_errors::emitter::Handler;
use leo_errors::Result;
#[cfg(test)]
mod test;
/// Creates a new AST from a given file path and source code text.
pub fn parse_ast<T: AsRef<str>, Y: AsRef<str>>(path: T, source: Y) -> Result<Ast> {
Ok(Ast::new(parser::parse(path.as_ref(), source.as_ref())?))
pub fn parse_ast<T: AsRef<str>, Y: AsRef<str>>(handler: &Handler, path: T, source: Y) -> Result<Ast> {
Ok(Ast::new(parser::parse(handler, path.as_ref(), source.as_ref())?))
}
/// Parses program input from from the input file path and state file path
pub fn parse_program_input<T: AsRef<str>, Y: AsRef<str>, T2: AsRef<str>, Y2: AsRef<str>>(
input_string: T,
input_path: Y,
state_string: T2,
state_path: Y2,
) -> Result<Input> {
let input_syntax_tree = LeoInputParser::parse_file(input_string.as_ref()).map_err(|mut e| {
e.set_path(
input_path.as_ref(),
&input_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
let state_syntax_tree = LeoInputParser::parse_file(state_string.as_ref()).map_err(|mut e| {
e.set_path(
state_path.as_ref(),
&state_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
let mut input = Input::new();
input.parse_input(input_syntax_tree).map_err(|mut e| {
e.set_path(
input_path.as_ref(),
&input_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
input.parse_state(state_syntax_tree).map_err(|mut e| {
e.set_path(
state_path.as_ref(),
&state_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
Ok(input)
}

150
parser/src/parser/context.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -14,42 +14,48 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use std::{borrow::Cow, unimplemented};
use crate::{assert_no_whitespace, tokenizer::*, Token, KEYWORD_TOKENS};
use leo_ast::*;
use leo_errors::{LeoError, ParserError, Result, Span};
use leo_errors::emitter::Handler;
use leo_errors::{LeoError, ParserError, Result};
use leo_span::{Span, Symbol};
use std::{borrow::Cow, unimplemented};
use tendril::format_tendril;
/// Stores a program in tokenized format plus additional context.
/// May be converted into a [`Program`] AST by parsing all tokens.
pub struct ParserContext {
pub struct ParserContext<'a> {
#[allow(dead_code)]
pub(crate) handler: &'a Handler,
tokens: Vec<SpannedToken>,
end_span: Span,
// true if parsing an expression for an if statement -- means circuit inits are not legal
pub(crate) fuzzy_struct_state: bool,
}
impl Iterator for ParserContext {
impl Iterator for ParserContext<'_> {
type Item = SpannedToken;
fn next(&mut self) -> Option<SpannedToken> {
self.tokens.pop()
self.bump()
}
}
impl ParserContext {
impl<'a> ParserContext<'a> {
///
/// Returns a new [`ParserContext`] type given a vector of tokens.
///
pub fn new(mut tokens: Vec<SpannedToken>) -> Self {
pub fn new(handler: &'a Handler, mut tokens: Vec<SpannedToken>) -> Self {
tokens.reverse();
// todo: performance optimization here: drain filter
tokens = tokens
.into_iter()
.filter(|x| !matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_)))
.collect();
ParserContext {
Self {
handler,
end_span: tokens
.iter()
.find(|x| !x.span.content.trim().is_empty())
@ -60,6 +66,16 @@ impl ParserContext {
}
}
/// Returns the current token if there is one.
pub fn curr(&self) -> Option<&SpannedToken> {
self.tokens.last()
}
/// Emit the error `err`.
pub(crate) fn emit_err(&self, err: ParserError) {
self.handler.emit_err(err.into());
}
///
/// Returns an unexpected end of function [`SyntaxError`].
///
@ -75,12 +91,15 @@ impl ParserContext {
}
///
/// Returns a reference to the next token or error if it does not exist.
/// Returns a reference to the next SpannedToken or error if it does not exist.
///
pub fn peek(&self) -> Result<&SpannedToken> {
self.tokens.last().ok_or_else(|| self.eof())
self.curr().ok_or_else(|| self.eof())
}
///
/// Returns a reference to the next Token.
///
pub fn peek_token(&self) -> Cow<'_, Token> {
self.tokens
.last()
@ -112,14 +131,19 @@ impl ParserContext {
!self.tokens.is_empty()
}
/// Advances the current token.
pub fn bump(&mut self) -> Option<SpannedToken> {
self.tokens.pop()
}
///
/// Removes the next token if it exists and returns it, or [None] if
/// the next token does not exist.
///
pub fn eat(&mut self, token: Token) -> Option<SpannedToken> {
if let Some(SpannedToken { token: inner, .. }) = self.tokens.last() {
if let Some(SpannedToken { token: inner, .. }) = self.curr() {
if &token == inner {
return self.tokens.pop();
return self.bump();
}
}
None
@ -139,13 +163,12 @@ impl ParserContext {
pub fn eat_identifier(&mut self) -> Option<Identifier> {
if let Some(SpannedToken {
token: Token::Ident(_), ..
}) = self.tokens.last()
}) = self.curr()
{
let token = self.tokens.pop().unwrap();
if let SpannedToken {
token: Token::Ident(name),
span,
} = token
} = self.bump().unwrap()
{
return Some(Identifier { name, span });
} else {
@ -186,6 +209,21 @@ impl ParserContext {
})
}
/// Returns `true` if the next token is Function or if it is a Const followed by Function.
/// Returns `false` otherwise.
pub fn peek_is_function(&self) -> Result<bool> {
let first = &self.peek()?.token;
let next = if self.tokens.len() >= 2 {
&self.peek_next()?.token
} else {
return Ok(false);
};
Ok(matches!(
(first, next),
(Token::Function | Token::At, _) | (Token::Const, Token::Function)
))
}
///
/// Removes the next two tokens if they are a pair of [`GroupCoordinate`] and returns them,
/// or [None] if the next token is not a [`GroupCoordinate`].
@ -263,13 +301,12 @@ impl ParserContext {
pub fn eat_int(&mut self) -> Option<(PositiveNumber, Span)> {
if let Some(SpannedToken {
token: Token::Int(_), ..
}) = self.tokens.last()
}) = self.curr()
{
let token = self.tokens.pop().unwrap();
if let SpannedToken {
token: Token::Int(value),
span,
} = token
} = self.bump().unwrap()
{
return Some((PositiveNumber { value }, span));
} else {
@ -284,9 +321,9 @@ impl ParserContext {
/// the next token does not exist.
///
pub fn eat_any(&mut self, token: &[Token]) -> Option<SpannedToken> {
if let Some(SpannedToken { token: inner, .. }) = self.tokens.last() {
if let Some(SpannedToken { token: inner, .. }) = self.curr() {
if token.iter().any(|x| x == inner) {
return self.tokens.pop();
return self.bump();
}
}
None
@ -296,9 +333,9 @@ impl ParserContext {
/// Returns the span of the next token if it is equal to the given [`Token`], or error.
///
pub fn expect(&mut self, token: Token) -> Result<Span> {
if let Some(SpannedToken { token: inner, span }) = self.tokens.last() {
if let Some(SpannedToken { token: inner, span }) = self.curr() {
if &token == inner {
Ok(self.tokens.pop().unwrap().span)
Ok(self.bump().unwrap().span)
} else {
Err(ParserError::unexpected(inner, token, span).into())
}
@ -311,9 +348,9 @@ impl ParserContext {
/// Returns the span of the next token if it is equal to one of the given [`Token`]s, or error.
///
pub fn expect_oneof(&mut self, token: &[Token]) -> Result<SpannedToken> {
if let Some(SpannedToken { token: inner, span }) = self.tokens.last() {
if let Some(SpannedToken { token: inner, span }) = self.curr() {
if token.iter().any(|x| x == inner) {
Ok(self.tokens.pop().unwrap())
Ok(self.bump().unwrap())
} else {
return Err(ParserError::unexpected(
inner,
@ -334,27 +371,25 @@ impl ParserContext {
pub fn expect_loose_identifier(&mut self) -> Result<Identifier> {
if let Some(token) = self.eat_any(KEYWORD_TOKENS) {
return Ok(Identifier {
name: token.token.to_string().into(),
name: token.token.keyword_to_symbol().unwrap(),
span: token.span,
});
}
if let Some((int, span)) = self.eat_int() {
return Ok(Identifier { name: int.value, span });
let name = Symbol::intern(&int.value);
return Ok(Identifier { name, span });
}
self.expect_ident()
}
///
/// Returns the [`Identifier`] of the next token if it is an [`Identifier`], or error.
///
pub fn expect_ident(&mut self) -> Result<Identifier> {
if let Some(SpannedToken { token: inner, span }) = self.tokens.last() {
if let Some(SpannedToken { token: inner, span }) = self.curr() {
if let Token::Ident(_) = inner {
let token = self.tokens.pop().unwrap();
if let SpannedToken {
token: Token::Ident(name),
span,
} = token
} = self.bump().unwrap()
{
Ok(Identifier { name, span })
} else {
@ -378,4 +413,53 @@ impl ParserContext {
Err(self.eof())
}
}
/// Parses a list of `T`s using `inner`
/// The opening and closing delimiters are `bra` and `ket`,
/// and elements in the list are separated by `sep`.
/// When `(list, true)` is returned, `sep` was a terminator.
pub(super) fn parse_list<T>(
&mut self,
open: Token,
close: Token,
sep: Token,
mut inner: impl FnMut(&mut Self) -> Result<Option<T>>,
) -> Result<(Vec<T>, bool, Span)> {
let mut list = Vec::new();
let mut trailing = false;
// Parse opening delimiter.
let open_span = self.expect(open)?;
while self.peek()?.token != close {
// Parse the element. We allow inner parser recovery through the `Option`.
if let Some(elem) = inner(self)? {
list.push(elem);
}
// Parse the separator.
if self.eat(sep.clone()).is_none() {
trailing = false;
break;
}
}
// Parse closing delimiter.
let close_span = self.expect(close)?;
Ok((list, trailing, open_span + close_span))
}
/// Parse a list separated by `,` and delimited by parens.
pub(super) fn parse_paren_comma_list<T>(
&mut self,
f: impl FnMut(&mut Self) -> Result<Option<T>>,
) -> Result<(Vec<T>, bool, Span)> {
self.parse_list(Token::LeftParen, Token::RightParen, Token::Comma, f)
}
/// Returns true if the current token is `(`.
pub(super) fn peek_is_left_par(&self) -> bool {
matches!(self.curr().map(|t| &t.token), Some(Token::LeftParen))
}
}

282
parser/src/parser/expression.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -14,11 +14,12 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use tendril::format_tendril;
use super::*;
use leo_errors::{ParserError, Result};
use leo_span::sym;
use super::*;
use tendril::format_tendril;
const INT_TYPES: &[Token] = &[
Token::I8,
@ -35,7 +36,7 @@ const INT_TYPES: &[Token] = &[
Token::Group,
];
impl ParserContext {
impl ParserContext<'_> {
///
/// Returns an [`Expression`] AST node if the next token is an expression.
/// Includes circuit init expressions.
@ -81,44 +82,45 @@ impl ParserContext {
Ok(expr)
}
///
/// Returns an [`Expression`] AST node if the next tokens represent
/// a binary or expression.
///
/// Otherwise, tries to parse the next token using [`parse_conjunctive_expression`].
///
pub fn parse_disjunctive_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_conjunctive_expression()?;
while self.eat(Token::Or).is_some() {
let right = self.parse_conjunctive_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: BinaryOperation::Or,
left: Box::new(expr),
right: Box::new(right),
})
/// Constructs a binary expression `left op right`.
fn bin_expr(left: Expression, right: Expression, op: BinaryOperation) -> Expression {
Expression::Binary(BinaryExpression {
span: left.span() + right.span(),
op,
left: Box::new(left),
right: Box::new(right),
})
}
/// Parses a left-associative binary expression `<left> token <right>` using `f` for left/right.
/// The `token` is translated to `op` in the AST.
fn parse_bin_expr(
&mut self,
token: Token,
op: BinaryOperation,
mut f: impl FnMut(&mut Self) -> Result<Expression>,
) -> Result<Expression> {
let mut expr = f(self)?;
while self.eat(token.clone()).is_some() {
expr = Self::bin_expr(expr, f(self)?, op);
}
Ok(expr)
}
/// Returns an [`Expression`] AST node if the next tokens represent
/// a binary or expression.
///
/// Otherwise, tries to parse the next token using [`parse_conjunctive_expression`].
pub fn parse_disjunctive_expression(&mut self) -> Result<Expression> {
self.parse_bin_expr(Token::Or, BinaryOperation::Or, Self::parse_conjunctive_expression)
}
/// Returns an [`Expression`] AST node if the next tokens represent a
/// binary and expression.
///
/// Otherwise, tries to parse the next token using [`parse_bit_or_expression`].
///
/// Otherwise, tries to parse the next token using [`parse_equality_expression`].
pub fn parse_conjunctive_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_equality_expression()?;
while self.eat(Token::And).is_some() {
let right = self.parse_equality_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: BinaryOperation::And,
left: Box::new(expr),
right: Box::new(right),
})
}
Ok(expr)
self.parse_bin_expr(Token::And, BinaryOperation::And, Self::parse_equality_expression)
}
///
@ -169,7 +171,7 @@ impl ParserContext {
///
// pub fn parse_bit_and_expression(&mut self) -> Result<Expression> {
// let mut expr = self.parse_equality_expression()?;
// while self.eat(Token::BitAnd).is_some() {
// while self.eat(Token::Ampersand).is_some() {
// let right = self.parse_equality_expression()?;
// expr = Expression::Binary(BinaryExpression {
// span: expr.span() + right.span(),
@ -181,53 +183,41 @@ impl ParserContext {
// Ok(expr)
// }
///
/// Returns an [`Expression`] AST node if the next tokens represent a
/// binary equals or not equals expression.
///
/// Otherwise, tries to parse the next token using [`parse_ordering_expression`].
///
pub fn parse_equality_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_ordering_expression()?;
if let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Eq, Token::NotEq]) {
let right = self.parse_ordering_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: match op {
Token::Eq => BinaryOperation::Eq,
Token::NotEq => BinaryOperation::Ne,
_ => unimplemented!(),
},
left: Box::new(expr),
right: Box::new(right),
})
let op = match op {
Token::Eq => BinaryOperation::Eq,
Token::NotEq => BinaryOperation::Ne,
_ => unimplemented!(),
};
expr = Self::bin_expr(expr, right, op);
}
Ok(expr)
}
///
/// Returns an [`Expression`] AST node if the next tokens represent a
/// binary relational expression: less than, less than or equals, greater than, greater than or equals.
///
/// Otherwise, tries to parse the next token using [`parse_shift_expression`].
///
pub fn parse_ordering_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_additive_expression()?;
while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Lt, Token::LtEq, Token::Gt, Token::GtEq])
{
let right = self.parse_additive_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: match op {
Token::Lt => BinaryOperation::Lt,
Token::LtEq => BinaryOperation::Le,
Token::Gt => BinaryOperation::Gt,
Token::GtEq => BinaryOperation::Ge,
_ => unimplemented!(),
},
left: Box::new(expr),
right: Box::new(right),
})
let op = match op {
Token::Lt => BinaryOperation::Lt,
Token::LtEq => BinaryOperation::Le,
Token::Gt => BinaryOperation::Gt,
Token::GtEq => BinaryOperation::Ge,
_ => unimplemented!(),
};
expr = Self::bin_expr(expr, right, op);
}
Ok(expr)
}
@ -257,76 +247,55 @@ impl ParserContext {
// Ok(expr)
// }
///
/// Returns an [`Expression`] AST node if the next tokens represent a
/// binary addition or subtraction expression.
///
/// Otherwise, tries to parse the next token using [`parse_mul_div_pow_expression`].
///
pub fn parse_additive_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_multiplicative_expression()?;
while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Add, Token::Minus]) {
let right = self.parse_multiplicative_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: match op {
Token::Add => BinaryOperation::Add,
Token::Minus => BinaryOperation::Sub,
_ => unimplemented!(),
},
left: Box::new(expr),
right: Box::new(right),
})
let op = match op {
Token::Add => BinaryOperation::Add,
Token::Minus => BinaryOperation::Sub,
_ => unimplemented!(),
};
expr = Self::bin_expr(expr, right, op);
}
Ok(expr)
}
///
/// Returns an [`Expression`] AST node if the next tokens represent a
/// binary multiplication, division, or modulus expression.
///
/// Otherwise, tries to parse the next token using [`parse_exponential_expression`].
///
pub fn parse_multiplicative_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_exponential_expression()?;
while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Mul, Token::Div]) {
let right = self.parse_exponential_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: match op {
Token::Mul => BinaryOperation::Mul,
Token::Div => BinaryOperation::Div,
// Token::Mod => BinaryOperation::Mod,
_ => unimplemented!(),
},
left: Box::new(expr),
right: Box::new(right),
})
let op = match op {
Token::Mul => BinaryOperation::Mul,
Token::Div => BinaryOperation::Div,
// Token::Mod => BinaryOperation::Mod,
_ => unimplemented!(),
};
expr = Self::bin_expr(expr, right, op);
}
Ok(expr)
}
///
/// Returns an [`Expression`] AST node if the next tokens represent a
/// binary exponentiation expression.
///
/// Otherwise, tries to parse the next token using [`parse_cast_expression`].
///
pub fn parse_exponential_expression(&mut self) -> Result<Expression> {
let mut exprs = vec![self.parse_cast_expression()?];
while self.eat(Token::Exp).is_some() {
exprs.push(self.parse_cast_expression()?);
}
let mut expr = exprs.remove(exprs.len() - 1);
while !exprs.is_empty() {
let sub_expr = exprs.remove(exprs.len() - 1);
expr = Expression::Binary(BinaryExpression {
span: expr.span() + sub_expr.span(),
op: BinaryOperation::Pow,
left: Box::new(sub_expr),
right: Box::new(expr),
})
let mut expr = self.parse_cast_expression()?;
if self.eat(Token::Exp).is_some() {
let right = self.parse_exponential_expression()?;
expr = Self::bin_expr(expr, right, BinaryOperation::Pow);
}
Ok(expr)
}
@ -401,21 +370,12 @@ impl ParserContext {
/// Otherwise, tries to parse the next token using [`parse_primary_expression`].
///
pub fn parse_postfix_expression(&mut self) -> Result<Expression> {
// We don't directly parse named-type's and Identifier's here as
// the ABNF states. Rather the primary expression already
// handle those. The ABNF is more specific for language reasons.
let mut expr = self.parse_primary_expression()?;
while let Some(token) = self.eat_any(&[
Token::LeftSquare,
Token::Dot,
Token::LeftParen,
Token::DoubleColon,
Token::LengthOf,
]) {
while let Some(token) = self.eat_any(&[Token::LeftSquare, Token::Dot, Token::LeftParen, Token::DoubleColon]) {
match token.token {
Token::LengthOf => {
expr = Expression::LengthOf(LengthOfExpression {
span: expr.span().clone(),
inner: Box::new(expr),
})
}
Token::LeftSquare => {
if self.eat(Token::DotDot).is_some() {
let right = if self.peek_token().as_ref() != &Token::RightSquare {
@ -425,12 +385,12 @@ impl ParserContext {
};
let end = self.expect(Token::RightSquare)?;
expr = Expression::ArrayRangeAccess(ArrayRangeAccessExpression {
expr = Expression::Access(AccessExpression::ArrayRange(ArrayRangeAccess {
span: expr.span() + &end,
array: Box::new(expr),
left: None,
right,
});
}));
continue;
}
@ -443,35 +403,35 @@ impl ParserContext {
};
let end = self.expect(Token::RightSquare)?;
expr = Expression::ArrayRangeAccess(ArrayRangeAccessExpression {
expr = Expression::Access(AccessExpression::ArrayRange(ArrayRangeAccess {
span: expr.span() + &end,
array: Box::new(expr),
left: Some(Box::new(left)),
right,
});
}));
} else {
let end = self.expect(Token::RightSquare)?;
expr = Expression::ArrayAccess(ArrayAccessExpression {
expr = Expression::Access(AccessExpression::Array(ArrayAccess {
span: expr.span() + &end,
array: Box::new(expr),
index: Box::new(left),
});
}));
}
}
Token::Dot => {
if let Some(ident) = self.eat_identifier() {
expr = Expression::CircuitMemberAccess(CircuitMemberAccessExpression {
expr = Expression::Access(AccessExpression::Member(MemberAccess {
span: expr.span() + &ident.span,
circuit: Box::new(expr),
inner: Box::new(expr),
name: ident,
type_: None,
});
}));
} else if let Some((num, span)) = self.eat_int() {
expr = Expression::TupleAccess(TupleAccessExpression {
expr = Expression::Access(AccessExpression::Tuple(TupleAccess {
span: expr.span() + &span,
tuple: Box::new(expr),
index: num,
});
}));
} else {
let next = self.peek()?;
return Err(ParserError::unexpected_str(&next.token, "int or ident", &next.span).into());
@ -481,8 +441,7 @@ impl ParserContext {
let mut arguments = Vec::new();
let end_span;
loop {
let end = self.eat(Token::RightParen);
if let Some(end) = end {
if let Some(end) = self.eat(Token::RightParen) {
end_span = end.span;
break;
}
@ -500,11 +459,12 @@ impl ParserContext {
}
Token::DoubleColon => {
let ident = self.expect_ident()?;
expr = Expression::CircuitStaticFunctionAccess(CircuitStaticFunctionAccessExpression {
expr = Expression::Access(AccessExpression::Static(StaticAccess {
span: expr.span() + &ident.span,
circuit: Box::new(expr),
inner: Box::new(expr),
type_: None,
name: ident,
});
}));
}
_ => unimplemented!(),
}
@ -526,39 +486,17 @@ impl ParserContext {
})
}
///
/// Returns an [`Expression`] AST node if the next tokens represent an
/// Returns an [`Expression`] AST node if the next tokens represent a
/// circuit initialization expression.
///
pub fn parse_circuit_expression(&mut self, identifier: Identifier) -> Result<Expression> {
self.expect(Token::LeftCurly)?;
let mut members = Vec::new();
let end_span;
loop {
if let Some(end) = self.eat(Token::RightCurly) {
end_span = end.span;
break;
}
let name = self.expect_ident()?;
if self.eat(Token::Colon).is_some() {
let expression = self.parse_expression()?;
members.push(CircuitImpliedVariableDefinition {
identifier: name,
expression: Some(expression),
});
} else {
members.push(CircuitImpliedVariableDefinition {
identifier: name.clone(),
expression: None,
});
}
if self.eat(Token::Comma).is_none() {
end_span = self.expect(Token::RightCurly)?;
break;
}
}
let (members, _, span) = self.parse_list(Token::LeftCurly, Token::RightCurly, Token::Comma, |p| {
Ok(Some(CircuitImpliedVariableDefinition {
identifier: p.expect_ident()?,
expression: p.eat(Token::Colon).map(|_| p.parse_expression()).transpose()?,
}))
})?;
Ok(Expression::CircuitInit(CircuitInitExpression {
span: &identifier.span + &end_span,
span: &identifier.span + &span,
name: identifier,
members,
}))
@ -617,8 +555,8 @@ impl ParserContext {
let first = self.parse_spread_or_expression()?;
if self.eat(Token::Semicolon).is_some() {
let dimensions = self
.parse_array_dimensions()?
.ok_or_else(|| ParserError::unable_to_parse_array_dimensions(span))?;
.parse_array_dimensions()
.map_err(|_| ParserError::unable_to_parse_array_dimensions(span))?;
let end = self.expect(Token::RightSquare)?;
let first = match first {
SpreadOrExpression::Spread(first) => {
@ -723,7 +661,7 @@ impl ParserContext {
}
Token::BigSelf => {
let ident = Identifier {
name: token.to_string().into(),
name: sym::SelfUpper,
span,
};
if !self.fuzzy_struct_state && self.peek_token().as_ref() == &Token::LeftCurly {
@ -732,17 +670,15 @@ impl ParserContext {
Expression::Identifier(ident)
}
}
Token::Input | Token::LittleSelf => {
let ident = Identifier {
name: token.to_string().into(),
span,
};
Expression::Identifier(ident)
}
// Token::LengthOf => Expression::LengthOf(LengthOfExpression {
// span,
// inner: Box::new(self.parse_primary_expression()?),
// }),
Token::LittleSelf => Expression::Identifier(Identifier {
name: sym::SelfLower,
span,
}),
Token::Input => Expression::Identifier(Identifier { name: sym::input, span }),
t if crate::type_::TYPE_TOKENS.contains(&t) => Expression::Identifier(Identifier {
name: t.keyword_to_symbol().unwrap(),
span,
}),
token => {
return Err(ParserError::unexpected_str(token, "expression", &span).into());
}

392
parser/src/parser/file.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -14,15 +14,13 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use tendril::format_tendril;
use leo_errors::{ParserError, Result, Span};
use super::*;
use crate::KEYWORD_TOKENS;
use super::*;
use leo_errors::{ParserError, Result};
use leo_span::{sym, Span};
impl ParserContext {
impl ParserContext<'_> {
///
/// Returns a [`Program`] AST if all tokens can be consumed and represent a valid Leo program.
///
@ -41,42 +39,38 @@ impl ParserContext {
import_statements.push(self.parse_import_statement()?);
}
Token::Circuit => {
self.expect(Token::Circuit)?;
let (id, circuit) = self.parse_circuit()?;
circuits.insert(id, circuit);
}
Token::Function | Token::At => {
Token::Ident(ident) => match *ident {
sym::test => return Err(ParserError::test_function(&token.span).into()),
kw @ (sym::Struct | sym::Class) => {
self.emit_err(ParserError::unexpected(kw, "circuit", &token.span));
self.bump().unwrap();
let (id, circuit) = self.parse_circuit()?;
circuits.insert(id, circuit);
}
_ => return Err(Self::unexpected_item(token).into()),
},
// Const functions share the first token with the global Const.
Token::Const if self.peek_is_function()? => {
let (id, function) = self.parse_function_declaration()?;
functions.insert(id, function);
}
Token::Ident(ident) if ident.as_ref() == "test" => {
return Err(ParserError::test_function(&token.span).into());
}
Token::Const => {
let (name, global_const) = self.parse_global_const_declaration()?;
global_consts.insert(name, global_const);
}
Token::Function | Token::At => {
let (id, function) = self.parse_function_declaration()?;
functions.insert(id, function);
}
Token::Type => {
let (name, alias) = self.parse_type_alias()?;
aliases.insert(name, alias);
}
_ => {
return Err(ParserError::unexpected(
&token.token,
[
Token::Import,
Token::Circuit,
Token::Function,
Token::Ident("test".into()),
Token::At,
]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
.join(", "),
&token.span,
)
.into());
}
_ => return Err(Self::unexpected_item(token).into()),
}
}
Ok(Program {
@ -91,58 +85,46 @@ impl ParserContext {
})
}
///
fn unexpected_item(token: &SpannedToken) -> ParserError {
ParserError::unexpected(
&token.token,
[
Token::Import,
Token::Circuit,
Token::Function,
Token::Ident(sym::test),
Token::At,
]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
.join(", "),
&token.span,
)
}
/// Returns an [`Annotation`] AST node if the next tokens represent a supported annotation.
///
pub fn parse_annotation(&mut self) -> Result<Annotation> {
let start = self.expect(Token::At)?;
let name = self.expect_ident()?;
if name.name.as_ref() == "context" {
return Err(ParserError::context_annotation(&name.span).into());
}
let name = self.parse_annotation_name()?;
assert_no_whitespace(&start, &name.span, &name.name, "@")?;
assert_no_whitespace(&start, &name.span, &name.name.as_str(), "@")?;
let end_span;
let arguments = if self.eat(Token::LeftParen).is_some() {
let mut args = Vec::new();
let mut comma = false;
loop {
if let Some(end) = self.eat(Token::RightParen) {
if comma {
return Err(ParserError::unexpected(
Token::RightParen,
[Token::Ident("identifier".into()), Token::Int("number".into())]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
.join(", "),
&end.span,
)
.into());
}
end_span = end.span;
break;
}
comma = false;
if let Some(ident) = self.eat_identifier() {
args.push(ident.name);
} else if let Some((int, _)) = self.eat_int() {
args.push(int.value);
let (end_span, arguments) = if self.peek_is_left_par() {
let (args, _, span) = self.parse_paren_comma_list(|p| {
Ok(if let Some(ident) = p.eat_identifier() {
Some(ident.name)
} else if let Some((int, _)) = p.eat_int() {
Some(Symbol::intern(&int.value))
} else {
let token = self.peek()?;
return Err(ParserError::unexpected_str(&token.token, "ident or int", &token.span).into());
}
if self.eat(Token::Comma).is_none() && !comma {
end_span = self.expect(Token::RightParen)?;
break;
}
comma = true;
}
args
let token = p.expect_any()?;
p.emit_err(ParserError::unexpected_str(&token.token, "ident or int", &token.span));
None
})
})?;
(span, args)
} else {
end_span = name.span.clone();
Vec::new()
(name.span.clone(), Vec::new())
};
Ok(Annotation {
name,
@ -151,24 +133,26 @@ impl ParserContext {
})
}
///
/// Returns a vector of [`PackageAccess`] AST nodes if the next tokens represent package access
/// expressions within an import statement.
///
pub fn parse_package_accesses(&mut self, span: &Span) -> Result<Vec<PackageAccess>> {
let mut out = Vec::new();
self.expect(Token::LeftParen)?;
while self.eat(Token::RightParen).is_none() {
let access = self.parse_package_access()?;
out.push(access);
if self.eat(Token::Comma).is_none() {
self.expect(Token::RightParen)?;
break;
}
/// Parses `foo` in an annotation `@foo . That is, the name of the annotation.
fn parse_annotation_name(&mut self) -> Result<Identifier> {
let mut name = self.expect_ident()?;
// Recover `context` instead of `test`.
if name.name == sym::context {
self.emit_err(ParserError::context_annotation(&name.span));
name.name = sym::test;
}
Ok(name)
}
/// Returns a vector of [`PackageAccess`] AST nodes if the next tokens represent package access
/// expressions within an import statement.
pub fn parse_package_accesses(&mut self, span: &Span) -> Result<Vec<PackageAccess>> {
let (out, ..) = self.parse_paren_comma_list(|p| p.parse_package_access().map(Some))?;
if out.is_empty() {
return Err(ParserError::invalid_import_list(span).into());
self.emit_err(ParserError::invalid_import_list(span));
}
Ok(out)
@ -191,7 +175,7 @@ impl ParserContext {
name.span = name.span + span;
let next = self.expect_ident()?;
name.span = name.span + next.span;
name.name = format_tendril!("{}-{}", name.name, next.name);
name.name = Symbol::intern(&format!("{}-{}", name.name, next.name));
}
if self.peek_token().as_ref() == &Token::Dot {
@ -220,9 +204,7 @@ impl ParserContext {
}
}
///
/// Returns an [`Identifier`] AST node if the next tokens represent a valid package name.
///
pub fn parse_package_name(&mut self) -> Result<Identifier> {
// Build the package name, starting with valid characters up to a dash `-` (Token::Minus).
let mut base = self.expect_loose_identifier()?;
@ -234,22 +216,22 @@ impl ParserContext {
let span = self.expect(Token::Minus)?;
base.span = base.span + span;
let next = self.expect_loose_identifier()?;
base.name = format_tendril!("{}-{}", base.name, next.name);
base.name = Symbol::intern(&format!("{}-{}", base.name, next.name));
base.span = base.span + next.span;
}
Token::Int(_) => {
let (num, span) = self.eat_int().unwrap();
base.name = format_tendril!("{}{}", base.name, num.value);
base.name = Symbol::intern(&format!("{}{}", base.name, num.value));
base.span = base.span + span;
}
Token::Ident(_) => {
let next = self.expect_ident()?;
base.name = format_tendril!("{}{}", base.name, next.name);
base.name = Symbol::intern(&format!("{}{}", base.name, next.name));
base.span = base.span + next.span;
}
x if KEYWORD_TOKENS.contains(x) => {
let next = self.expect_loose_identifier()?;
base.name = format_tendril!("{}{}", base.name, next.name);
base.name = Symbol::intern(&format!("{}{}", base.name, next.name));
base.span = base.span + next.span;
}
_ => break,
@ -257,17 +239,18 @@ impl ParserContext {
}
// Return an error if the package name contains a keyword.
if let Some(token) = KEYWORD_TOKENS.iter().find(|x| x.to_string() == base.name.as_ref()) {
return Err(ParserError::unexpected_str(token, "package name", &base.span).into());
if let Some(token) = KEYWORD_TOKENS.iter().find(|x| x.keyword_to_symbol() == Some(base.name)) {
self.emit_err(ParserError::unexpected_str(token, "package name", &base.span));
}
// Return an error if the package name contains invalid characters.
if !base
.name
.as_str()
.chars()
.all(|x| x.is_ascii_lowercase() || x.is_ascii_digit() || x == '-' || x == '_')
{
return Err(ParserError::invalid_package_name(&base.span).into());
self.emit_err(ParserError::invalid_package_name(&base.span));
}
// Return the package name.
@ -281,7 +264,7 @@ impl ParserContext {
pub fn parse_package_path(&mut self) -> Result<PackageOrPackages> {
let package_name = self.parse_package_name()?;
self.expect(Token::Dot)?;
if self.peek()?.token == Token::LeftParen {
if self.peek_is_left_par() {
let accesses = self.parse_package_accesses(&package_name.span)?;
Ok(PackageOrPackages::Packages(Packages {
span: &package_name.span + accesses.last().map(|x| x.span()).unwrap_or(&package_name.span),
@ -311,83 +294,112 @@ impl ParserContext {
})
}
///
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member variable
/// or circuit member function.
///
pub fn parse_circuit_declaration(&mut self) -> Result<Vec<CircuitMember>> {
let mut members = Vec::new();
let peeked = &self.peek()?.token;
let mut last_variable = peeked == &Token::Function || peeked == &Token::At;
let (mut semi_colons, mut commas) = (false, false);
while self.eat(Token::RightCurly).is_none() {
if !last_variable {
let (variable, last) = self.parse_member_variable_declaration()?;
members.push(variable);
let peeked = &self.peek()?;
if peeked.token == Token::Semicolon {
if commas {
return Err(ParserError::mixed_commas_and_semicolons(&peeked.span).into());
}
semi_colons = true;
self.expect(Token::Semicolon)?;
} else {
if semi_colons {
return Err(ParserError::mixed_commas_and_semicolons(&peeked.span).into());
}
commas = true;
self.eat(Token::Comma);
}
if last {
last_variable = last;
}
members.push(if self.peek_is_function()? {
// function
self.parse_member_function_declaration()?
} else if *self.peek_token() == Token::Static {
// static const
self.parse_const_member_variable_declaration()?
} else {
let function = self.parse_member_function_declaration()?;
members.push(function);
}
// variable
let variable = self.parse_member_variable_declaration()?;
if let Some(semi) = self.eat(Token::Semicolon) {
if commas {
self.emit_err(ParserError::mixed_commas_and_semicolons(&semi.span));
}
semi_colons = true;
}
if let Some(comma) = self.eat(Token::Comma) {
if semi_colons {
self.emit_err(ParserError::mixed_commas_and_semicolons(&comma.span));
}
commas = true;
}
variable
});
}
self.ban_mixed_member_order(&members);
Ok(members)
}
/// Emits errors if order isn't `consts variables functions`.
fn ban_mixed_member_order(&self, members: &[CircuitMember]) {
let mut had_var = false;
let mut had_fun = false;
for member in members {
match member {
CircuitMember::CircuitConst(id, _, e) if had_var => {
self.emit_err(ParserError::member_const_after_var(&(id.span() + e.span())));
}
CircuitMember::CircuitConst(id, _, e) if had_fun => {
self.emit_err(ParserError::member_const_after_fun(&(id.span() + e.span())));
}
CircuitMember::CircuitVariable(id, _) if had_fun => {
self.emit_err(ParserError::member_var_after_fun(id.span()));
}
CircuitMember::CircuitConst(..) => {}
CircuitMember::CircuitVariable(..) => had_var = true,
CircuitMember::CircuitFunction(..) => had_fun = true,
}
}
}
/// Parses `IDENT: TYPE`.
fn parse_typed_field_name(&mut self) -> Result<(Identifier, Type)> {
let name = self.expect_ident()?;
self.expect(Token::Colon)?;
let type_ = self.parse_type()?.0;
Ok((name, type_))
}
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member static constant.
pub fn parse_const_member_variable_declaration(&mut self) -> Result<CircuitMember> {
self.expect(Token::Static)?;
self.expect(Token::Const)?;
// `IDENT: TYPE = EXPR`:
let (name, type_) = self.parse_typed_field_name()?;
self.expect(Token::Assign)?;
let literal = self.parse_primary_expression()?;
self.expect(Token::Semicolon)?;
Ok(CircuitMember::CircuitConst(name, type_, literal))
}
///
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member variable.
///
pub fn parse_member_variable_declaration(&mut self) -> Result<(CircuitMember, bool)> {
let name = self.expect_ident()?;
self.expect(Token::Colon)?;
let type_ = self.parse_type()?.0;
pub fn parse_member_variable_declaration(&mut self) -> Result<CircuitMember> {
let (name, type_) = self.parse_typed_field_name()?;
let peeked = &self.peek()?.token;
if peeked == &Token::Function || peeked == &Token::At || peeked == &Token::RightCurly {
return Ok((CircuitMember::CircuitVariable(name, type_), true));
} else if peeked == &Token::Comma || peeked == &Token::Semicolon {
let peeked = &self.peek_next()?.token;
if peeked == &Token::Function || peeked == &Token::At || peeked == &Token::RightCurly {
return Ok((CircuitMember::CircuitVariable(name, type_), true));
}
}
Ok((CircuitMember::CircuitVariable(name, type_), false))
Ok(CircuitMember::CircuitVariable(name, type_))
}
///
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member function.
///
pub fn parse_member_function_declaration(&mut self) -> Result<CircuitMember> {
let peeked = self.peek()?.clone();
if peeked.token == Token::Function || peeked.token == Token::At {
if self.peek_is_function()? {
let function = self.parse_function_declaration()?;
Ok(CircuitMember::CircuitFunction(function.1))
Ok(CircuitMember::CircuitFunction(Box::new(function.1)))
} else {
return Err(ParserError::unexpected(
&peeked.token,
[Token::Function, Token::At]
[Token::Function, Token::At, Token::Const]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
@ -403,8 +415,18 @@ impl ParserContext {
/// circuit name and definition statement.
///
pub fn parse_circuit(&mut self) -> Result<(Identifier, Circuit)> {
self.expect(Token::Circuit)?;
let name = self.expect_ident()?;
let name = if let Some(ident) = self.eat_identifier() {
ident
} else if let Some(scalar_type) = self.eat_any(crate::type_::TYPE_TOKENS) {
Identifier {
name: scalar_type.token.keyword_to_symbol().unwrap(),
span: scalar_type.span,
}
} else {
let next = self.peek()?;
return Err(ParserError::unexpected_str(&next.token, "ident", &next.span).into());
};
self.expect(Token::LeftCurly)?;
let members = self.parse_circuit_declaration()?;
@ -412,7 +434,6 @@ impl ParserContext {
name.clone(),
Circuit {
circuit_name: name,
core_mapping: std::cell::RefCell::new(None),
members,
},
))
@ -424,24 +445,26 @@ impl ParserContext {
pub fn parse_function_parameters(&mut self) -> Result<FunctionInput> {
let const_ = self.eat(Token::Const);
let mutable = self.eat(Token::Mut);
let reference = self.eat(Token::Ampersand);
let mut name = if let Some(token) = self.eat(Token::LittleSelf) {
Identifier {
name: token.token.to_string().into(),
name: sym::SelfLower,
span: token.span,
}
} else {
self.expect_ident()?
};
if name.name.as_ref() == "self" {
if name.name == sym::SelfLower {
if let Some(mutable) = &mutable {
// Handle `mut self`.
name.span = &mutable.span + &name.span;
name.name = "mut self".to_string().into();
return Ok(FunctionInput::MutSelfKeyword(MutSelfKeyword { identifier: name }));
self.emit_err(ParserError::mut_self_parameter(&(&mutable.span + &name.span)));
return Ok(Self::build_ref_self(name, mutable));
} else if let Some(reference) = &reference {
// Handle `&self`.
return Ok(Self::build_ref_self(name, reference));
} else if let Some(const_) = &const_ {
// Handle `const self`.
name.span = &const_.span + &name.span;
name.name = "const self".to_string().into();
name.name = Symbol::intern("const self");
return Ok(FunctionInput::ConstSelfKeyword(ConstSelfKeyword { identifier: name }));
}
// Handle `self`.
@ -449,7 +472,7 @@ impl ParserContext {
}
if let Some(mutable) = &mutable {
return Err(ParserError::mut_function_input(&(&mutable.span + &name.span)).into());
self.emit_err(ParserError::mut_function_input(&(&mutable.span + &name.span)));
}
self.expect(Token::Colon)?;
@ -463,42 +486,55 @@ impl ParserContext {
}))
}
///
/// Builds a function parameter `&self`.
fn build_ref_self(mut name: Identifier, reference: &SpannedToken) -> FunctionInput {
name.span = &reference.span + &name.span;
// FIXME(Centril): This should be *two* tokens, NOT one!
name.name = Symbol::intern("&self");
FunctionInput::RefSelfKeyword(RefSelfKeyword { identifier: name })
}
/// Returns an [`(Identifier, Function)`] AST node if the next tokens represent a function name
/// and function definition.
///
pub fn parse_function_declaration(&mut self) -> Result<(Identifier, Function)> {
let mut annotations = Vec::new();
// Parse any annotations.
let mut annotations = IndexMap::new();
while self.peek_token().as_ref() == &Token::At {
annotations.push(self.parse_annotation()?);
let annotation = self.parse_annotation()?;
annotations.insert(annotation.name.name, annotation);
}
// Parse optional const modifier.
let const_ = self.eat(Token::Const).is_some();
// Parse `function IDENT`.
let start = self.expect(Token::Function)?;
let name = self.expect_ident()?;
self.expect(Token::LeftParen)?;
let mut inputs = Vec::new();
while self.eat(Token::RightParen).is_none() {
let input = self.parse_function_parameters()?;
inputs.push(input);
if self.eat(Token::Comma).is_none() {
self.expect(Token::RightParen)?;
break;
}
}
// Parse parameters.
let (inputs, ..) = self.parse_paren_comma_list(|p| p.parse_function_parameters().map(Some))?;
// Parse return type.
let output = if self.eat(Token::Arrow).is_some() {
Some(self.parse_type()?.0)
} else {
None
};
// Parse the function body.
let block = self.parse_block()?;
Ok((
name.clone(),
Function {
annotations,
identifier: name,
input: inputs,
const_,
output,
span: start + block.span.clone(),
block,
core_mapping: <_>::default(),
},
))
}
@ -513,7 +549,7 @@ impl ParserContext {
.variable_names
.iter()
.map(|variable_name| variable_name.identifier.clone())
.collect::<Vec<Identifier>>();
.collect();
Ok((variable_names, statement))
}

23
parser/src/parser/mod.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -19,6 +19,16 @@
//! This module contains the [`parse()`] method which calls the underlying [`tokenize()`]
//! method to create a new program ast.
use crate::{tokenizer::*, Token};
use leo_ast::*;
use leo_errors::emitter::Handler;
use leo_errors::{ParserError, Result};
use leo_span::{Span, Symbol};
use indexmap::IndexMap;
use std::unimplemented;
mod context;
pub use context::*;
@ -27,13 +37,6 @@ pub mod file;
pub mod statement;
pub mod type_;
use std::unimplemented;
use crate::{tokenizer::*, Token};
use indexmap::IndexMap;
use leo_ast::*;
use leo_errors::{ParserError, Result, Span};
pub(crate) fn assert_no_whitespace(left_span: &Span, right_span: &Span, left: &str, right: &str) -> Result<()> {
if left_span.col_stop != right_span.col_start {
let mut error_span = left_span + right_span;
@ -46,8 +49,8 @@ pub(crate) fn assert_no_whitespace(left_span: &Span, right_span: &Span, left: &s
}
/// Creates a new program from a given file path and source code text.
pub fn parse(path: &str, source: &str) -> Result<Program> {
let mut tokens = ParserContext::new(crate::tokenize(path, source.into())?);
pub fn parse(handler: &Handler, path: &str, source: &str) -> Result<Program> {
let mut tokens = ParserContext::new(handler, crate::tokenize(path, source.into())?);
tokens.parse_program()
}

163
parser/src/parser/statement.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -17,6 +17,7 @@
use super::*;
use leo_errors::{ParserError, Result};
use leo_span::sym;
const ASSIGN_TOKENS: &[Token] = &[
Token::Assign,
@ -36,7 +37,7 @@ const ASSIGN_TOKENS: &[Token] = &[
// Token::AndEq,
];
impl ParserContext {
impl ParserContext<'_> {
///
/// Returns an [`Identifier`] AST node if the given [`Expression`] AST node evaluates to an
/// identifier access. The access is stored in the given accesses.
@ -44,25 +45,29 @@ impl ParserContext {
pub fn construct_assignee_access(expr: Expression, accesses: &mut Vec<AssigneeAccess>) -> Result<Identifier> {
let identifier;
match expr {
Expression::CircuitMemberAccess(expr) => {
identifier = Self::construct_assignee_access(*expr.circuit, accesses)?;
accesses.push(AssigneeAccess::Member(expr.name));
}
Expression::TupleAccess(expr) => {
identifier = Self::construct_assignee_access(*expr.tuple, accesses)?;
accesses.push(AssigneeAccess::Tuple(expr.index, expr.span));
}
Expression::ArrayRangeAccess(expr) => {
identifier = Self::construct_assignee_access(*expr.array, accesses)?;
accesses.push(AssigneeAccess::ArrayRange(
expr.left.map(|x| *x),
expr.right.map(|x| *x),
));
}
Expression::ArrayAccess(expr) => {
identifier = Self::construct_assignee_access(*expr.array, accesses)?;
accesses.push(AssigneeAccess::ArrayIndex(*expr.index));
}
Expression::Access(access) => match access {
AccessExpression::Member(expr) => {
identifier = Self::construct_assignee_access(*expr.inner, accesses)?;
accesses.push(AssigneeAccess::Member(expr.name));
}
AccessExpression::Tuple(expr) => {
identifier = Self::construct_assignee_access(*expr.tuple, accesses)?;
accesses.push(AssigneeAccess::Tuple(expr.index, expr.span));
}
AccessExpression::ArrayRange(expr) => {
identifier = Self::construct_assignee_access(*expr.array, accesses)?;
accesses.push(AssigneeAccess::ArrayRange(
expr.left.map(|x| *x),
expr.right.map(|x| *x),
));
}
AccessExpression::Array(expr) => {
identifier = Self::construct_assignee_access(*expr.array, accesses)?;
accesses.push(AssigneeAccess::ArrayIndex(*expr.index));
}
_ => return Err(ParserError::invalid_assignment_target(access.span()).into()),
},
Expression::Identifier(id) => identifier = id,
_ => return Err(ParserError::invalid_assignment_target(expr.span()).into()),
}
@ -177,9 +182,7 @@ impl ParserContext {
})
}
///
/// Returns a [`ConditionalStatement`] AST node if the next tokens represent a conditional statement.
///
pub fn parse_conditional_statement(&mut self) -> Result<ConditionalStatement> {
let start = self.expect(Token::If)?;
self.fuzzy_struct_state = true;
@ -188,12 +191,10 @@ impl ParserContext {
let body = self.parse_block()?;
let next = if self.eat(Token::Else).is_some() {
let s = self.parse_statement()?;
match s {
Statement::Block(_) | Statement::Conditional(_) => Some(Box::new(s)),
s => {
return Err(ParserError::unexpected_statement(&s, "Block or Conditional", s.span()).into());
}
if !matches!(s, Statement::Block(_) | Statement::Conditional(_)) {
self.emit_err(ParserError::unexpected_statement(&s, "Block or Conditional", s.span()));
}
Some(Box::new(s))
} else {
None
};
@ -206,19 +207,20 @@ impl ParserContext {
})
}
///
/// Returns an [`IterationStatement`] AST node if the next tokens represent an iteration statement.
///
pub fn parse_loop_statement(&mut self) -> Result<IterationStatement> {
let start_span = self.expect(Token::For)?;
let ident = self.expect_ident()?;
self.expect(Token::In)?;
// Parse iteration range.
let start = self.parse_expression()?;
self.expect(Token::DotDot)?;
let inclusive = self.eat(Token::Assign).is_some();
self.fuzzy_struct_state = true;
let stop = self.parse_conditional_expression()?;
self.fuzzy_struct_state = false;
let block = self.parse_block()?;
Ok(IterationStatement {
@ -231,59 +233,56 @@ impl ParserContext {
})
}
///
/// Returns a [`ConsoleArgs`] AST node if the next tokens represent a formatted string.
///
pub fn parse_console_args(&mut self) -> Result<ConsoleArgs> {
let start_span;
let string = match self.expect_any()? {
SpannedToken {
token: Token::StringLit(chars),
span,
} => {
start_span = span;
chars
let mut string = None;
let (parameters, _, span) = self.parse_paren_comma_list(|p| {
if string.is_none() {
let SpannedToken { token, span } = p.expect_any()?;
string = Some(match token {
Token::StringLit(chars) => chars,
_ => {
p.emit_err(ParserError::unexpected_str(token, "formatted string", &span));
Vec::new()
}
});
Ok(None)
} else {
p.parse_expression().map(Some)
}
SpannedToken { token, span } => {
return Err(ParserError::unexpected_str(token, "formatted string", &span).into());
}
};
// let parts = FormatStringPart::from_string(string);
let mut parameters = Vec::new();
while self.eat(Token::Comma).is_some() {
let param = self.parse_expression()?;
parameters.push(param);
}
})?;
Ok(ConsoleArgs {
string,
span: &start_span + parameters.last().map(|x| x.span()).unwrap_or(&start_span),
string: string.unwrap_or_default(),
span,
parameters,
})
}
///
/// Returns a [`ConsoleStatement`] AST node if the next tokens represent a console statement.
///
pub fn parse_console_statement(&mut self) -> Result<ConsoleStatement> {
let keyword = self.expect(Token::Console)?;
self.expect(Token::Dot)?;
let function = self.expect_ident()?;
self.expect(Token::LeftParen)?;
let function = match &*function.name {
"assert" => {
let function = match function.name {
sym::assert => {
self.expect(Token::LeftParen)?;
let expr = self.parse_expression()?;
self.expect(Token::RightParen)?;
ConsoleFunction::Assert(expr)
}
"error" => ConsoleFunction::Error(self.parse_console_args()?),
"log" => ConsoleFunction::Log(self.parse_console_args()?),
sym::error => ConsoleFunction::Error(self.parse_console_args()?),
sym::log => ConsoleFunction::Log(self.parse_console_args()?),
x => {
return Err(ParserError::unexpected_ident(x, &["assert", "error", "log"], &function.span).into());
// Not sure what it is, assume it's `log`.
self.emit_err(ParserError::unexpected_ident(
x,
&["assert", "error", "log"],
&function.span,
));
ConsoleFunction::Log(self.parse_console_args()?)
}
};
self.expect(Token::RightParen)?;
self.expect(Token::Semicolon)?;
Ok(ConsoleStatement {
@ -292,14 +291,13 @@ impl ParserContext {
})
}
///
/// Returns a [`VariableName`] AST node if the next tokens represent a variable name with
/// valid keywords.
///
pub fn parse_variable_name(&mut self, span: &SpannedToken) -> Result<VariableName> {
let mutable = self.eat(Token::Mut);
if let Some(mutable) = &mutable {
return Err(ParserError::let_mut_statement(&(&mutable.span + &span.span)).into());
self.emit_err(ParserError::let_mut_statement(&(&mutable.span + &span.span)));
}
let name = self.expect_ident()?;
@ -310,35 +308,24 @@ impl ParserContext {
})
}
///
/// Returns a [`DefinitionStatement`] AST node if the next tokens represent a definition statement.
///
pub fn parse_definition_statement(&mut self) -> Result<DefinitionStatement> {
let declare = self.expect_oneof(&[Token::Let, Token::Const])?;
let mut variable_names = Vec::new();
let next = self.eat(Token::LeftParen);
variable_names.push(self.parse_variable_name(&declare)?);
if next.is_some() {
let mut eaten_ending = false;
while self.eat(Token::Comma).is_some() {
if self.eat(Token::RightParen).is_some() {
eaten_ending = true;
break;
}
variable_names.push(self.parse_variable_name(&declare)?);
}
if !eaten_ending {
self.expect(Token::RightParen)?;
}
}
let type_ = if self.eat(Token::Colon).is_some() {
Some(self.parse_type()?.0)
// Parse variable names.
let variable_names = if self.peek_is_left_par() {
self.parse_paren_comma_list(|p| p.parse_variable_name(&declare).map(Some))
.map(|(vars, ..)| vars)?
} else {
None
vec![self.parse_variable_name(&declare)?]
};
// Parse an optional type ascription.
let type_ = self
.eat(Token::Colon)
.map(|_| self.parse_type().map(|t| t.0))
.transpose()?;
self.expect(Token::Assign)?;
let expr = self.parse_expression()?;
self.expect(Token::Semicolon)?;

77
parser/src/parser/type_.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -17,7 +17,7 @@
use super::*;
use leo_errors::{ParserError, Result};
const TYPE_TOKENS: &[Token] = &[
pub(crate) const TYPE_TOKENS: &[Token] = &[
Token::I8,
Token::I16,
Token::I32,
@ -35,7 +35,7 @@ const TYPE_TOKENS: &[Token] = &[
Token::Char,
];
impl ParserContext {
impl ParserContext<'_> {
///
/// Returns a [`IntegerType`] AST node if the given token is a supported integer type, or [`None`].
///
@ -55,58 +55,51 @@ impl ParserContext {
})
}
///
/// Returns an [`ArrayDimensions`] AST node if the next tokens represent dimensions for an array type.
///
pub fn parse_array_dimensions(&mut self) -> Result<Option<ArrayDimensions>> {
Ok(if let Some((int, _)) = self.eat_int() {
Some(ArrayDimensions(vec![int]))
} else if self.eat(Token::Underscore).is_some() {
None
pub fn parse_array_dimensions(&mut self) -> Result<ArrayDimensions> {
Ok(if let Some(dim) = self.parse_array_dimension() {
dim
} else {
self.expect(Token::LeftParen)?;
let mut dimensions = Vec::new();
loop {
if let Some((int, _)) = self.eat_int() {
dimensions.push(int);
let mut had_item_err = false;
let (dims, _, span) = self.parse_paren_comma_list(|p| {
Ok(if let Some(dim) = p.parse_array_dimension() {
Some(dim)
} else {
let token = self.peek()?;
return Err(ParserError::unexpected_str(&token.token, "int", &token.span).into());
}
if self.eat(Token::Comma).is_none() {
break;
}
let token = p.expect_any()?;
p.emit_err(ParserError::unexpected_str(&token.token, "int", &token.span));
had_item_err = true;
None
})
})?;
if dims.is_empty() && !had_item_err {
self.emit_err(ParserError::array_tuple_dimensions_empty(&span));
}
self.expect(Token::RightParen)?;
Some(ArrayDimensions(dimensions))
ArrayDimensions::Multi(dims)
})
}
///
/// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type. Also
/// returns the span of the parsed token.
///
/// Parses a basic array dimension, i.e., an integer or `_`.
fn parse_array_dimension(&mut self) -> Option<ArrayDimensions> {
if let Some((int, _)) = self.eat_int() {
Some(ArrayDimensions::Number(int))
} else if self.eat(Token::Underscore).is_some() {
Some(ArrayDimensions::Unspecified)
} else {
None
}
}
/// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type.
/// Also returns the span of the parsed token.
pub fn parse_type(&mut self) -> Result<(Type, Span)> {
Ok(if let Some(token) = self.eat(Token::BigSelf) {
(Type::SelfType, token.span)
} else if let Some(ident) = self.eat_identifier() {
let span = ident.span.clone();
(Type::Identifier(ident), span)
} else if let Some(token) = self.eat(Token::LeftParen) {
let mut types = Vec::new();
let end_span;
loop {
if let Some(end) = self.eat(Token::RightParen) {
end_span = end.span;
break;
}
types.push(self.parse_type()?.0);
if self.eat(Token::Comma).is_none() {
end_span = self.expect(Token::RightParen)?;
break;
}
}
(Type::Tuple(types), token.span + end_span)
} else if self.peek_is_left_par() {
let (types, _, span) = self.parse_paren_comma_list(|p| p.parse_type().map(|t| Some(t.0)))?;
(Type::Tuple(types), span)
} else if let Some(token) = self.eat(Token::LeftSquare) {
let (inner, _) = self.parse_type()?;
self.expect(Token::Semicolon)?;

186
parser/src/test.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -14,17 +14,18 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::{tokenizer, ParserContext, SpannedToken};
use leo_ast::{Expression, ExpressionStatement, Statement, ValueExpression};
use leo_errors::Span;
use leo_errors::{emitter::Handler, LeoError};
use leo_span::{symbol::create_session_if_not_set_then, Span};
use leo_test_framework::{
runner::{Namespace, ParseType, Runner},
Test,
};
use serde::Serialize;
use serde_yaml::Value;
use tokenizer::Token;
use crate::{tokenizer, ParserContext};
struct TokenNamespace;
impl Namespace for TokenNamespace {
@ -33,18 +34,19 @@ impl Namespace for TokenNamespace {
}
fn run_test(&self, test: Test) -> Result<Value, String> {
let output = tokenizer::tokenize("test", test.content.into());
output
.map(|tokens| {
Value::String(
tokens
.into_iter()
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(","),
)
})
.map_err(|x| x.to_string())
create_session_if_not_set_then(|_| {
tokenizer::tokenize("test", test.content.into())
.map(|tokens| {
Value::String(
tokens
.into_iter()
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join(","),
)
})
.map_err(|x| x.to_string())
})
}
}
@ -60,6 +62,37 @@ fn not_fully_consumed(tokens: &mut ParserContext) -> Result<(), String> {
Err(out)
}
fn with_handler<T>(
tokens: Vec<SpannedToken>,
logic: impl FnOnce(&mut ParserContext<'_>) -> Result<T, LeoError>,
) -> Result<T, String> {
let (handler, buf) = Handler::new_with_buf();
let mut tokens = ParserContext::new(&handler, tokens);
let parsed = handler
.extend_if_error(logic(&mut tokens))
.map_err(|_| buf.extract().to_string())?;
not_fully_consumed(&mut tokens)?;
Ok(parsed)
}
fn implcit_value_expr() -> Expression {
Expression::Value(ValueExpression::Implicit("".into(), Span::default()))
}
fn tokenize(test: Test) -> Result<Vec<SpannedToken>, String> {
tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())
}
fn all_are_comments(tokens: &[SpannedToken]) -> bool {
tokens
.iter()
.all(|x| matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_)))
}
fn yaml_or_fail<T: Serialize>(value: T) -> Value {
serde_yaml::to_value(value).expect("serialization failed")
}
struct ParseExpressionNamespace;
impl Namespace for ParseExpressionNamespace {
@ -68,23 +101,13 @@ impl Namespace for ParseExpressionNamespace {
}
fn run_test(&self, test: Test) -> Result<Value, String> {
let tokenizer = tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())?;
if tokenizer
.iter()
.all(|x| matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_)))
{
return Ok(serde_yaml::to_value(&Expression::Value(ValueExpression::Implicit(
"".into(),
Span::default(),
)))
.expect("serialization failed"));
}
let mut tokens = ParserContext::new(tokenizer);
let parsed = tokens.parse_expression().map_err(|x| x.to_string())?;
not_fully_consumed(&mut tokens)?;
Ok(serde_yaml::to_value(&parsed).expect("serialization failed"))
create_session_if_not_set_then(|_| {
let tokenizer = tokenize(test)?;
if all_are_comments(&tokenizer) {
return Ok(yaml_or_fail(implcit_value_expr()));
}
with_handler(tokenizer, |p| p.parse_expression()).map(yaml_or_fail)
})
}
}
@ -96,23 +119,16 @@ impl Namespace for ParseStatementNamespace {
}
fn run_test(&self, test: Test) -> Result<Value, String> {
let tokenizer = tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())?;
if tokenizer
.iter()
.all(|x| matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_)))
{
return Ok(serde_yaml::to_value(&Statement::Expression(ExpressionStatement {
expression: Expression::Value(ValueExpression::Implicit("".into(), Span::default())),
span: Span::default(),
}))
.expect("serialization failed"));
}
let mut tokens = ParserContext::new(tokenizer);
let parsed = tokens.parse_statement().map_err(|x| x.to_string())?;
not_fully_consumed(&mut tokens)?;
Ok(serde_yaml::to_value(&parsed).expect("serialization failed"))
create_session_if_not_set_then(|_| {
let tokenizer = tokenize(test)?;
if all_are_comments(&tokenizer) {
return Ok(yaml_or_fail(Statement::Expression(ExpressionStatement {
expression: implcit_value_expr(),
span: Span::default(),
})));
}
with_handler(tokenizer, |p| p.parse_statement()).map(yaml_or_fail)
})
}
}
@ -124,13 +140,72 @@ impl Namespace for ParseNamespace {
}
fn run_test(&self, test: Test) -> Result<Value, String> {
let tokenizer = tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())?;
let mut tokens = ParserContext::new(tokenizer);
create_session_if_not_set_then(|_| with_handler(tokenize(test)?, |p| p.parse_program()).map(yaml_or_fail))
}
}
let parsed = tokens.parse_program().map_err(|x| x.to_string())?;
not_fully_consumed(&mut tokens)?;
struct SerializeNamespace;
Ok(serde_yaml::to_value(&parsed).expect("serialization failed"))
// Helper functions to recursively filter keys from AST JSON.
// Redeclaring here since we don't want to make this public.
fn remove_key_from_json(value: &mut serde_json::Value, key: &str) {
match value {
serde_json::value::Value::Object(map) => {
map.remove(key);
for val in map.values_mut() {
remove_key_from_json(val, key);
}
}
serde_json::value::Value::Array(values) => {
for val in values.iter_mut() {
remove_key_from_json(val, key);
}
}
_ => (),
}
}
// Helper function to normalize AST
// Redeclaring here because we don't want to make this public
fn normalize_json_value(value: serde_json::Value) -> serde_json::Value {
match value {
serde_json::Value::Array(vec) => {
let orig_length = vec.len();
let mut new_vec: Vec<serde_json::Value> = vec
.into_iter()
.filter(|v| !matches!(v, serde_json::Value::Object(map) if map.is_empty()))
.map(normalize_json_value)
.collect();
if orig_length == 2 && new_vec.len() == 1 {
new_vec.pop().unwrap()
} else {
serde_json::Value::Array(new_vec)
}
}
serde_json::Value::Object(map) => {
serde_json::Value::Object(map.into_iter().map(|(k, v)| (k, normalize_json_value(v))).collect())
}
_ => value,
}
}
impl Namespace for SerializeNamespace {
fn parse_type(&self) -> ParseType {
ParseType::Whole
}
fn run_test(&self, test: Test) -> Result<Value, String> {
create_session_if_not_set_then(|_| {
let tokenizer = tokenize(test)?;
let parsed = with_handler(tokenizer, |p| p.parse_program())?;
let mut json = serde_json::to_value(parsed).expect("failed to convert to json value");
remove_key_from_json(&mut json, "span");
json = normalize_json_value(json);
Ok(serde_json::from_value::<serde_yaml::Value>(json).expect("failed serialization"))
})
}
}
@ -143,6 +218,7 @@ impl Runner for TestRunner {
"ParseStatement" => Box::new(ParseStatementNamespace),
"ParseExpression" => Box::new(ParseExpressionNamespace),
"Token" => Box::new(TokenNamespace),
"Serialize" => Box::new(SerializeNamespace),
_ => return None,
})
}

12
parser/src/tokenizer/lexer.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -15,7 +15,8 @@
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::tokenizer::{Char, Token};
use leo_errors::Span;
use leo_span::{Span, Symbol};
use serde::{Deserialize, Serialize};
use tendril::StrTendril;
@ -352,7 +353,7 @@ impl Token {
// else if let Some(len) = eat(input, "&=") {
// return (len, Some(Token::BitAndEq));
// }
// return (1, Some(Token::BitAnd));
return (1, Some(Token::Ampersand));
}
b'(' => return (1, Some(Token::LeftParen)),
b')' => return (1, Some(Token::RightParen)),
@ -388,9 +389,6 @@ impl Token {
return (len, Some(Token::DotDotDot));
} else if let Some(len) = eat(input, "..") {
return (len, Some(Token::DotDot));
} else if let Some(len) = eat(input, ".len()") {
// FIXME: remove this code once we allow method calls
return (len, Some(Token::LengthOf));
}
return (1, Some(Token::Dot));
}
@ -526,7 +524,7 @@ impl Token {
"u32" => Token::U32,
"u64" => Token::U64,
"u128" => Token::U128,
_ => Token::Ident(ident),
_ => Token::Ident(Symbol::intern(&ident)),
}),
);
}

104
parser/src/tokenizer/mod.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -28,7 +28,8 @@ pub(crate) use self::token::*;
pub(crate) mod lexer;
pub(crate) use self::lexer::*;
use leo_errors::{LeoError, ParserError, Span};
use leo_errors::{LeoError, ParserError};
use leo_span::Span;
use tendril::StrTendril;
@ -112,30 +113,31 @@ pub(crate) fn tokenize(path: &str, input: StrTendril) -> Result<Vec<SpannedToken
#[cfg(test)]
mod tests {
use super::*;
use leo_span::symbol::create_session_if_not_set_then;
#[test]
fn test_tokenizer() {
// &
// &=
// |
// |=
// ^
// ^=
// ~
// <<
// <<=
// >>
// >>=
// >>>
// >>>=
// %
// %=
// ||=
// &&=
create_session_if_not_set_then(|_| {
// &=
// |
// |=
// ^
// ^=
// ~
// <<
// <<=
// >>
// >>=
// >>>
// >>>=
// %
// %=
// ||=
// &&=
let tokens = tokenize(
"test_path",
r#"
let tokens = tokenize(
"test_path",
r#"
"test"
"test{}test"
"test{}"
@ -170,6 +172,7 @@ mod tests {
input
let
mut
&
return
static
string
@ -223,25 +226,27 @@ mod tests {
// test
/* test */
//"#
.into(),
)
.unwrap();
let mut output = String::new();
for SpannedToken { token, .. } in tokens.iter() {
output += &format!("{} ", token.to_string());
}
.into(),
)
.unwrap();
let mut output = String::new();
for SpannedToken { token, .. } in tokens.iter() {
output += &format!("{} ", token.to_string());
}
// & &= | |= ^ ^= ~ << <<= >> >>= >>> >>>= % %= ||= &&=
assert_eq!(
output,
r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
// & &= | |= ^ ^= ~ << <<= >> >>= >>> >>>= % %= ||= &&=
assert_eq!(
output,
r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut & return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
/* test */ // "#
);
);
});
}
#[test]
fn test_spans() {
let raw = r#"
create_session_if_not_set_then(|_| {
let raw = r#"
test
// test
test
@ -251,20 +256,21 @@ mod tests {
test */
test
"#;
let tokens = tokenize("test_path", raw.into()).unwrap();
let mut line_indicies = vec![0];
for (i, c) in raw.chars().enumerate() {
if c == '\n' {
line_indicies.push(i + 1);
let tokens = tokenize("test_path", raw.into()).unwrap();
let mut line_indicies = vec![0];
for (i, c) in raw.chars().enumerate() {
if c == '\n' {
line_indicies.push(i + 1);
}
}
}
for token in tokens.iter() {
let token_raw = token.token.to_string();
let start = line_indicies.get(token.span.line_start - 1).unwrap();
let stop = line_indicies.get(token.span.line_stop - 1).unwrap();
let original = &raw[*start + token.span.col_start - 1..*stop + token.span.col_stop - 1];
assert_eq!(original, &token_raw);
}
// println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
for token in tokens.iter() {
let token_raw = token.token.to_string();
let start = line_indicies.get(token.span.line_start - 1).unwrap();
let stop = line_indicies.get(token.span.line_stop - 1).unwrap();
let original = &raw[*start + token.span.col_start - 1..*stop + token.span.col_stop - 1];
assert_eq!(original, &token_raw);
}
// println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
})
}
}

75
parser/src/tokenizer/token.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use leo_span::{sym, Symbol};
use serde::{Deserialize, Serialize};
use std::fmt;
use tendril::StrTendril;
@ -48,14 +50,14 @@ impl fmt::Display for Char {
pub enum Token {
// Lexical Grammar
// Literals
CommentLine(#[serde(with = "leo_errors::common::tendril_json")] StrTendril),
CommentBlock(#[serde(with = "leo_errors::common::tendril_json")] StrTendril),
CommentLine(#[serde(with = "leo_span::tendril_json")] StrTendril),
CommentBlock(#[serde(with = "leo_span::tendril_json")] StrTendril),
StringLit(Vec<leo_ast::Char>),
Ident(#[serde(with = "leo_errors::common::tendril_json")] StrTendril),
Int(#[serde(with = "leo_errors::common::tendril_json")] StrTendril),
Ident(Symbol),
Int(#[serde(with = "leo_span::tendril_json")] StrTendril),
True,
False,
AddressLit(#[serde(with = "leo_errors::common::tendril_json")] StrTendril),
AddressLit(#[serde(with = "leo_span::tendril_json")] StrTendril),
CharLit(Char),
// Symbols
@ -127,6 +129,7 @@ pub enum Token {
As,
Circuit,
Console,
/// Const variable and a const function.
Const,
Else,
For,
@ -135,16 +138,14 @@ pub enum Token {
In,
Let,
Mut,
/// Represents `&`.
/// Used for `Reference` and `BitAnd`.
Ampersand,
Return,
Static,
Type,
// Not yet in ABNF
// arr.len() token - hacky zone
LengthOf,
// Not yet in ABNF
// BitAnd,
// BitAndEq,
// BitOr,
// BitOrEq,
@ -192,13 +193,13 @@ pub const KEYWORD_TOKENS: &[Token] = &[
Token::Input,
Token::Let,
Token::Mut,
Token::Ampersand,
Token::Return,
Token::BigSelf,
Token::LittleSelf,
Token::Static,
Token::True,
Token::Type,
Token::LengthOf,
Token::U8,
Token::U16,
Token::U32,
@ -207,11 +208,52 @@ pub const KEYWORD_TOKENS: &[Token] = &[
];
impl Token {
///
/// Returns `true` if the `self` token equals a Leo keyword.
///
pub fn is_keyword(&self) -> bool {
KEYWORD_TOKENS.iter().any(|x| x == self)
KEYWORD_TOKENS.contains(self)
}
/// Converts `self` to the corresponding `Symbol` if it `is_keyword`.
pub fn keyword_to_symbol(&self) -> Option<Symbol> {
Some(match self {
Token::Address => sym::address,
Token::As => sym::As,
Token::Bool => sym::bool,
Token::Char => sym::char,
Token::Circuit => sym::circuit,
Token::Console => sym::console,
Token::Const => sym::Const,
Token::Else => sym::Else,
Token::False => sym::False,
Token::Field => sym::field,
Token::For => sym::For,
Token::Function => sym::function,
Token::Group => sym::group,
Token::I8 => sym::i8,
Token::I16 => sym::i16,
Token::I32 => sym::i32,
Token::I64 => sym::i64,
Token::I128 => sym::i128,
Token::If => sym::If,
Token::Import => sym::import,
Token::In => sym::In,
Token::Input => sym::input,
Token::Let => sym::Let,
Token::Mut => sym::Mut,
Token::Ampersand => sym::Ampersand,
Token::Return => sym::Return,
Token::BigSelf => sym::SelfUpper,
Token::LittleSelf => sym::SelfLower,
Token::Static => sym::Static,
Token::True => sym::True,
Token::Type => sym::Type,
Token::U8 => sym::u8,
Token::U16 => sym::u16,
Token::U32 => sym::u32,
Token::U64 => sym::u64,
Token::U128 => sym::u128,
_ => return None,
})
}
}
@ -307,12 +349,11 @@ impl fmt::Display for Token {
In => write!(f, "in"),
Let => write!(f, "let"),
Mut => write!(f, "mut"),
Ampersand => write!(f, "&"), // Used for `Reference` and `BitAnd`
Return => write!(f, "return"),
Static => write!(f, "static"),
Type => write!(f, "type"),
LengthOf => write!(f, ".len()"), // FIXME
Eof => write!(f, ""),
// BitAnd => write!(f, "&"),
// BitAndEq => write!(f, "&="),
// BitOr => write!(f, "|"),
// BitOrEq => write!(f, "|="),

17
parser/tests/mod.rs
View File

@ -1,17 +0,0 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
mod serialization;

1162
parser/tests/serialization/expected_leo_ast/linear_regression.json
View File

File diff suppressed because it is too large Load Diff

100
parser/tests/serialization/expected_leo_ast/one_plus_one.json
View File

@ -1,100 +0,0 @@
{
"name": "",
"expected_input": [],
"import_statements": [],
"imports": {},
"aliases": {},
"circuits": {},
"global_consts": {},
"functions": {
"{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main() -> u8 {\\\"}\"}": {
"annotations": [],
"identifier": "{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main() -> u8 {\\\"}\"}",
"input": [],
"output": {
"IntegerType": "U8"
},
"block": {
"statements": [
{
"Return": {
"expression": {
"Binary": {
"left": {
"Value": {
"Integer": [
"U8",
"1",
{
"span" : {
"line_start": 2,
"line_stop": 2,
"col_start": 12,
"col_stop": 15,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
]
}
},
"right": {
"Value": {
"Integer": [
"U8",
"1",
{
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 18,
"col_stop": 21,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
]
}
},
"op": "Add",
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 12,
"col_stop": 21,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 5,
"col_stop": 21,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
}
],
"span": {
"line_start": 1,
"line_stop": 3,
"col_start": 23,
"col_stop": 2,
"path": "",
"content": "function main() -> u8 {\n ...\n}"
}
},
"span": {
"line_start": 1,
"line_stop": 3,
"col_start": 1,
"col_stop": 2,
"path": "",
"content": "function main() -> u8 {\n ...\n}"
}
}
}
}

1189
parser/tests/serialization/expected_leo_ast/palindrome.json
View File

File diff suppressed because it is too large Load Diff

299
parser/tests/serialization/expected_leo_ast/pedersen_hash.json
View File

@ -1,299 +0,0 @@
{
"name": "",
"expected_input": [],
"import_statements": [],
"imports": {},
"aliases": {},
"circuits": {
"{\"name\":\"PedersenHash\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":9,\\\"col_stop\\\":21,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"circuit PedersenHash {\\\"}\"}": {
"circuit_name": "{\"name\":\"PedersenHash\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":9,\\\"col_stop\\\":21,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"circuit PedersenHash {\\\"}\"}",
"core_mapping": null,
"members": [
{
"CircuitVariable": [
"{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":2,\\\"line_stop\\\":2,\\\"col_start\\\":5,\\\"col_stop\\\":15,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" parameters: [group; 256];\\\"}\"}",
{
"Array": [
"Group",
[
{
"value": "256"
}
]
]
}
]
},
{
"CircuitFunction": {
"annotations": [],
"identifier": "{\"name\":\"new\",\"span\":\"{\\\"line_start\\\":5,\\\"line_stop\\\":5,\\\"col_start\\\":14,\\\"col_stop\\\":17,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function new(parameters: [group; 256]) -> Self {\\\"}\"}",
"input": [
{
"Variable": {
"identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":5,\\\"line_stop\\\":5,\\\"col_start\\\":18,\\\"col_stop\\\":28,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function new(parameters: [group; 256]) -> Self {\\\"}\"}",
"const_": false,
"mutable": true,
"type_": {
"Array": [
"Group",
[
{
"value": "256"
}
]
]
}
}
}
],
"output": "SelfType",
"block": {
"statements": [
{
"Return": {
"expression": {
"CircuitInit": {
"name": "{\"name\":\"Self\",\"span\":\"{\\\"line_start\\\":6,\\\"line_stop\\\":6,\\\"col_start\\\":16,\\\"col_stop\\\":20,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return Self { parameters: parameters };\\\"}\"}",
"members": [
{
"identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":6,\\\"line_stop\\\":6,\\\"col_start\\\":23,\\\"col_stop\\\":33,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return Self { parameters: parameters };\\\"}\"}",
"expression": {
"Identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":6,\\\"line_stop\\\":6,\\\"col_start\\\":35,\\\"col_stop\\\":45,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return Self { parameters: parameters };\\\"}\"}"
}
}
]
}
}
}
}
]
}
}
},
{
"CircuitFunction": {
"annotations": [],
"identifier": "{\"name\":\"hash\",\"span\":\"{\\\"line_start\\\":9,\\\"line_stop\\\":9,\\\"col_start\\\":14,\\\"col_stop\\\":18,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function hash(self, bits: [bool; 256]) -> group {\\\"}\"}",
"input": [
{
"SelfKeyword": "{\"name\":\"self\",\"span\":\"{\\\"line_start\\\":9,\\\"line_stop\\\":9,\\\"col_start\\\":19,\\\"col_stop\\\":23,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function hash(self, bits: [bool; 256]) -> group {\\\"}\"}"
},
{
"Variable": {
"identifier": "{\"name\":\"bits\",\"span\":\"{\\\"line_start\\\":9,\\\"line_stop\\\":9,\\\"col_start\\\":25,\\\"col_stop\\\":29,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function hash(self, bits: [bool; 256]) -> group {\\\"}\"}",
"const_": false,
"mutable": true,
"type_": {
"Array": [
"Boolean",
[
{
"value": "256"
}
]
]
}
}
}
],
"output": "Group",
"block": {
"statements": [
{
"Definition": {
"declaration_type": "Let",
"variable_names": [
{
"mutable": true,
"identifier": "{\"name\":\"digest\",\"span\":\"{\\\"line_start\\\":10,\\\"line_stop\\\":10,\\\"col_start\\\":13,\\\"col_stop\\\":19,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" let digest: group = 0group;\\\"}\"}"
}
],
"type_": "Group",
"value": {
"Value": {
"Group": {
"Single": "0"
}
}
}
}
},
{
"Iteration": {
"variable": "{\"name\":\"i\",\"span\":\"{\\\"line_start\\\":11,\\\"line_stop\\\":11,\\\"col_start\\\":13,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" for i in 0..256 {\\\"}\"}",
"start": {
"Value": {
"Implicit": "0"
}
},
"stop": {
"Value": {
"Implicit": "256"
}
},
"inclusive": false,
"block": {
"statements": [
{
"Conditional": {
"condition": {
"ArrayAccess": {
"array": {
"Identifier": "{\"name\":\"bits\",\"span\":\"{\\\"line_start\\\":12,\\\"line_stop\\\":12,\\\"col_start\\\":16,\\\"col_stop\\\":20,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" if bits[i] {\\\"}\"}"
},
"index": {
"Identifier": "{\"name\":\"i\",\"span\":\"{\\\"line_start\\\":12,\\\"line_stop\\\":12,\\\"col_start\\\":21,\\\"col_stop\\\":22,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" if bits[i] {\\\"}\"}"
}
}
},
"block": {
"statements": [
{
"Assign": {
"operation": "Add",
"assignee": {
"identifier": "{\"name\":\"digest\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":17,\\\"col_stop\\\":23,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}",
"accesses": []
},
"value": {
"ArrayAccess": {
"array": {
"CircuitMemberAccess": {
"circuit": {
"Identifier": "{\"name\":\"self\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":27,\\\"col_stop\\\":31,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}"
},
"name": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":32,\\\"col_stop\\\":42,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}",
"type_": null
}
},
"index": {
"Identifier": "{\"name\":\"i\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":43,\\\"col_stop\\\":44,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}"
}
}
}
}
}
]
},
"next": null
}
}
]
}
}
},
{
"Return": {
"expression": {
"Identifier": "{\"name\":\"digest\",\"span\":\"{\\\"line_start\\\":16,\\\"line_stop\\\":16,\\\"col_start\\\":16,\\\"col_stop\\\":22,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return digest;\\\"}\"}"
}
}
}
]
}
}
}
]
}
},
"global_consts": {},
"functions": {
"{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}": {
"annotations": [],
"identifier": "{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}",
"input": [
{
"Variable": {
"identifier": "{\"name\":\"hash_input\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":15,\\\"col_stop\\\":25,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}",
"const_": false,
"mutable": true,
"type_": {
"Array": [
"Boolean",
[
{
"value": "256"
}
]
]
}
}
},
{
"Variable": {
"identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":46,\\\"col_stop\\\":56,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}",
"const_": true,
"mutable": false,
"type_": {
"Array": [
"Group",
[
{
"value": "256"
}
]
]
}
}
}
],
"output": "Group",
"block": {
"statements": [
{
"Definition": {
"declaration_type": "Const",
"variable_names": [
{
"mutable": false,
"identifier": "{\"name\":\"pedersen\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":11,\\\"col_stop\\\":19,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
}
],
"type_": null,
"value": {
"Call": {
"function": {
"CircuitStaticFunctionAccess": {
"circuit": {
"Identifier": "{\"name\":\"PedersenHash\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":22,\\\"col_stop\\\":34,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
},
"name": "{\"name\":\"new\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":36,\\\"col_stop\\\":39,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
}
},
"arguments": [
{
"Identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":40,\\\"col_stop\\\":50,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
}
]
}
}
}
},
{
"Return": {
"expression": {
"Call": {
"function": {
"CircuitMemberAccess": {
"circuit": {
"Identifier": "{\"name\":\"pedersen\",\"span\":\"{\\\"line_start\\\":23,\\\"line_stop\\\":23,\\\"col_start\\\":12,\\\"col_stop\\\":20,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return pedersen.hash(hash_input);\\\"}\"}"
},
"name": "{\"name\":\"hash\",\"span\":\"{\\\"line_start\\\":23,\\\"line_stop\\\":23,\\\"col_start\\\":21,\\\"col_stop\\\":25,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return pedersen.hash(hash_input);\\\"}\"}",
"type_": null
}
},
"arguments": [
{
"Identifier": "{\"name\":\"hash_input\",\"span\":\"{\\\"line_start\\\":23,\\\"line_stop\\\":23,\\\"col_start\\\":26,\\\"col_stop\\\":36,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return pedersen.hash(hash_input);\\\"}\"}"
}
]
}
}
}
}
]
}
}
}
}

1209
parser/tests/serialization/expected_leo_ast/silly_sudoku.json
View File

File diff suppressed because it is too large Load Diff

6
parser/tests/serialization/json.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
@ -17,7 +17,7 @@
use leo_ast::Ast;
#[cfg(not(feature = "ci_skip"))]
use leo_ast::Program;
use leo_errors::{LeoError, Result};
use leo_errors::{emitter::Handler, LeoError, Result};
use std::fs::File;
use std::io::BufReader;
@ -28,7 +28,7 @@ fn to_ast(program_filepath: &Path) -> Result<Ast> {
let program_string = std::fs::read_to_string(program_filepath).expect("failed to open test");
// Parses the Leo file and constructs a leo ast.
leo_parser::parse_ast("", &program_string)
leo_parser::parse_ast(&Handler::default(), "", &program_string)
}
fn setup() {

7
parser/tests/serialization/leo/deprecated_error.leo
View File

@ -1,7 +0,0 @@
function main() {
return 1 + 1;
}
test function old {
console.log("old");
}

65
parser/tests/serialization/leo/linear_regression.leo
View File

@ -1,65 +0,0 @@
circuit Point {
x: i32,
y: i32,
function new(x: i32, y: i32) -> Self {
return Self { x, y };
}
}
circuit LinearRegression {
points: [Point; 5],
// Instantiates a linear regression circuit.
function new(points: [Point; 5]) -> Self {
return Self { points };
}
// Return the slope of the linear regression.
function slope(self) -> i32 {
let num_points = 5i32;
// Calculate the sums.
let x_sum = 0i32;
let y_sum = 0i32;
let xy_sum = 0i32;
let x2_sum = 0i32;
for i in 0..5 {
x_sum += self.points[i].x;
y_sum += self.points[i].y;
xy_sum += self.points[i].x * self.points[i].y;
x2_sum += self.points[i].x * self.points[i].x;
}
let numerator = (num_points * xy_sum) - (x_sum * y_sum);
let denominator = (num_points * x2_sum) - (x_sum * x_sum);
let slope = numerator / denominator;
return slope;
}
// Return the offset of the linear regression.
function offset(self, slope: i32) -> i32 {
let num_points = 5i32;
// Calculate the sum.
let x_sum = 0i32;
let y_sum = 0i32;
for i in 0..5 {
x_sum += self.points[i].x;
y_sum += self.points[i].y;
}
return (y_sum - slope * x_sum) / num_points;
}
}
function main (x: i32, y: i32) -> [i32; 2] {
let points: [Point; 5] = [
Point{x: x + 1, y: y + 1},
Point{x: x + 2, y: y + 2},
Point{x: x + 3, y: y + 3},
Point{x: x + 4, y: y + 4},
Point{x: x + 5, y: y + 5}
];
let reg = LinearRegression::new(points);
let slope = reg.slope();
let offset = reg.offset(slope);
return [slope, offset];
}

3
parser/tests/serialization/leo/one_plus_one.leo
View File

@ -1,3 +0,0 @@
function main() -> u8 {
return 1u8 + 1u8;
}

59
parser/tests/serialization/leo/palindrome.leo
View File

@ -1,59 +0,0 @@
// This Program takes in any 20-byte low register string and tells
// whether a string is a palindrome ignoring any spaces.
function main(str: [char; 20]) -> bool {
return is_palindrome(str);
}
function is_palindrome(str: [char; 20]) -> bool {
const str_len = 20u32; // saving const for convenience
// By default we assume that input is a palindrome.
let result = true;
let processed = 0u8;
for start in 0..(str_len / 2) {
let start_sym = str[start];
if start_sym != ' ' {
let skipped = 0u8;
let end_empty = 0u8;
let end_sym = ' ';
for end in (str_len - 1)..start {
if str[end] != ' ' && skipped == processed && end_sym == ' ' {
end_sym = str[end];
} else {
end_empty = end_empty + 1;
if str[end] != ' ' {
skipped = skipped + 1;
}
}
}
// If there are symbols left to the right from the start.
if end_sym != ' ' {
console.log("Comparing: {} ? {}", start_sym, end_sym);
if result {
result = (start_sym == end_sym);
}
processed = processed + 1;
}
}
}
console.log("Result is: {}", result);
return result;
}
@test
function test_is_palindrome() {
console.assert(is_palindrome("a b a "));
console.assert(is_palindrome("😀😀😀😀😀 😀😀😀😀😀"));
console.assert(is_palindrome("borrow or rob "));
console.assert(is_palindrome("bbbb aaaa aaaa bbbb"));
console.assert(is_palindrome("aaaaaaaaaaaaaaaaaaaa"));
console.assert(is_palindrome("taco cat "));
}

1
parser/tests/serialization/leo/parser_error.leo
View File

@ -1 +0,0 @@
invalid

25
parser/tests/serialization/leo/pedersen_hash.leo
View File

@ -1,25 +0,0 @@
circuit PedersenHash {
parameters: [group; 256];
// Instantiates a Pedersen hash circuit
function new(parameters: [group; 256]) -> Self {
return Self { parameters: parameters };
}
function hash(self, bits: [bool; 256]) -> group {
let digest: group = 0group;
for i in 0..256 {
if bits[i] {
digest += self.parameters[i];
}
}
return digest;
}
}
// The 'pedersen-hash' main function.
function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {
const pedersen = PedersenHash::new(parameters);
return pedersen.hash(hash_input);
}

71
parser/tests/serialization/leo/silly_sudoku.leo
View File

@ -1,71 +0,0 @@
import lib.SillySudoku;
// The `silly-sudoku` main function
function main(puzzle: [u8; (3, 3)], answer: [u8; (3, 3)]) -> bool {
console.log("Starting Sudoku solver...");
console.log("{}", puzzle);
// Instantiate the Sudoku puzzle.
let sudoku = SillySudoku { puzzle_grid: puzzle };
console.log("Checking Sudoku answer...");
console.log("{}", answer);
// Evaluate the Sudoku puzzle with the given answer.
let result = sudoku.solve(answer);
console.log("The answer is {}.", result);
return result;
}
// Tests that the `silly-sudoku` circuit outputs true on a correct answer.
@test
function test_solve_pass() {
let puzzle: [u8; (3, 3)] = [[0, 2, 0],
[0, 0, 6],
[0, 8, 9]];
let answer: [u8; (3, 3)] = [[1, 2, 3],
[4, 5, 6],
[7, 8, 9]];
// Runs the Sudoku checker.
let result = main(puzzle, answer);
// Expects the result to be true.
console.assert(true == result);
}
// Tests that the `silly-sudoku` circuit outputs false on an incorrect answer.
@test
function test_solve_fail() {
let puzzle: [u8; (3, 3)] = [[0, 2, 0],
[0, 0, 6],
[0, 8, 0]];
let answer: [u8; (3, 3)] = [[1, 2, 3],
[4, 5, 6],
[7, 8, 8]]; // We have an extra `8` in this column!
// Runs the Sudoku checker.
let result = main(puzzle, answer);
// Expects the result to be false.
console.assert(false == result);
}
// Test that the `silly-sudoku` circuit outputs the expected value on a custom test input.
@test(test_input)
function test_solve_with_input(
puzzle: [u8; (3, 3)],
answer: [u8; (3, 3)],
expected: bool
) {
// Runs the Sudoku checker.
let result = main(puzzle, answer);
console.log("expected {}, got {}", expected, result);
console.assert(expected == result);
}

17
parser/tests/serialization/mod.rs
View File

@ -1,17 +0,0 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
mod json;