AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2024-12-19 07:32:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

migrate parser

Browse Source
This commit is contained in:
gluax 2022-01-21 12:32:09 -08:00
parent 2a2df78603
commit 3593529c4d
30 changed files with 921 additions and 4916 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
parser/Cargo.toml
View File

@ -16,7 +16,7 @@ categories = [ "cryptography::cryptocurrencies", "web-programming" ]
include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ] include = [ "Cargo.toml", "src", "README.md", "LICENSE.md" ]
license = "GPL-3.0" license = "GPL-3.0"
edition = "2021" edition = "2021"
rust-version = "1.56.1" rust-version = "1.56"
[[bench]] [[bench]]
name = "leo_ast" name = "leo_ast"
@ -31,6 +31,14 @@ version = "1.5.3"
path = "../errors" path = "../errors"
version = "1.5.3" version = "1.5.3"
[dependencies.leo-input]
path = "../input"
version = "1.5.1"
[dependencies.leo-span]
path = "../span"
version = "1.5.3"
[dependencies.lazy_static] [dependencies.lazy_static]
version = "1.3.0" version = "1.3.0"

4
parser/README.md
View File

@ -30,7 +30,8 @@ Bolded ones are also keywords.
#### Symbols #### Symbols
- At - At
- Not - Not
- And - And (`&&`)
- Ampersand (`&`)
- Or - Or
- Eq - Eq
- NotEq - NotEq
@ -98,7 +99,6 @@ Bolded ones are also keywords.
- **If** - **If**
- **In** - **In**
- **Let** - **Let**
- **Mut**
- **Return** - **Return**
- **Static** - **Static**
- **String** - **String**

40
parser/benches/leo_ast.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -14,55 +14,51 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use leo_ast::Ast;
use leo_errors::emitter::Handler;
use leo_span::symbol::create_session_if_not_set_then;
use criterion::{criterion_group, criterion_main, Criterion}; use criterion::{criterion_group, criterion_main, Criterion};
use std::time::Duration; use std::time::Duration;
fn bench_big_if_else(c: &mut Criterion) { fn parse_ast(path: &str, input: &str) -> Ast {
let program_string = include_str!("./big_if_else.leo"); create_session_if_not_set_then(|_| {
let ast = leo_parser::parse_ast("./big_if_else.leo", program_string).expect("failed to parse benchmark"); leo_parser::parse_ast(&Handler::default(), path, input).expect("failed to parse benchmark")
})
}
fn bench_big_if_else(c: &mut Criterion) {
let ast = parse_ast("./big_if_else.leo", include_str!("./big_if_else.leo"));
c.bench_function("Ast::big_if_else", |b| b.iter(|| &ast)); c.bench_function("Ast::big_if_else", |b| b.iter(|| &ast));
} }
fn bench_big_ternary(c: &mut Criterion) { fn bench_big_ternary(c: &mut Criterion) {
let program_string = include_str!("./big_ternary.leo"); let ast = parse_ast("./big_ternary.leo", include_str!("./big_ternary.leo"));
let ast = leo_parser::parse_ast("./big_ternary.leo", program_string).expect("failed to parse benchmark");
c.bench_function("Ast::big_ternary", |b| b.iter(|| &ast)); c.bench_function("Ast::big_ternary", |b| b.iter(|| &ast));
} }
fn bench_big_circuit(c: &mut Criterion) { fn bench_big_circuit(c: &mut Criterion) {
let program_string = include_str!("./big_circuit.leo"); let ast = parse_ast("./big_circuit.leo", include_str!("./big_circuit.leo"));
let ast = leo_parser::parse_ast("./big_circuit.leo", program_string).expect("failed to parse benchmark");
c.bench_function("Ast::big_circuit", |b| b.iter(|| &ast)); c.bench_function("Ast::big_circuit", |b| b.iter(|| &ast));
} }
fn bench_long_expr(c: &mut Criterion) { fn bench_long_expr(c: &mut Criterion) {
let program_string = include_str!("./long_expr.leo"); let ast = parse_ast("./long_expr.leo", include_str!("./long_expr.leo"));
let ast = leo_parser::parse_ast("./long_expr.leo", program_string).expect("failed to parse benchmark");
c.bench_function("Ast::long_expr", |b| b.iter(|| &ast)); c.bench_function("Ast::long_expr", |b| b.iter(|| &ast));
} }
fn bench_long_array(c: &mut Criterion) { fn bench_long_array(c: &mut Criterion) {
let program_string = include_str!("./long_array.leo"); let ast = parse_ast("./long_array.leo", include_str!("./long_array.leo"));
let ast = leo_parser::parse_ast("./long_array.leo", program_string).expect("failed to parse benchmark");
c.bench_function("Ast::long_array", |b| b.iter(|| &ast)); c.bench_function("Ast::long_array", |b| b.iter(|| &ast));
} }
fn bench_many_foos(c: &mut Criterion) { fn bench_many_foos(c: &mut Criterion) {
let program_string = include_str!("./many_foos.leo"); let ast = parse_ast("./many_foos.leo", include_str!("./many_foos.leo"));
let ast = leo_parser::parse_ast("./many_foos.leo", program_string).expect("failed to parse benchmark");
c.bench_function("Ast::many_foos", |b| b.iter(|| &ast)); c.bench_function("Ast::many_foos", |b| b.iter(|| &ast));
} }
fn bench_many_assigns(c: &mut Criterion) { fn bench_many_assigns(c: &mut Criterion) {
let program_string = include_str!("./many_assigns.leo"); let ast = parse_ast("./many_assigns.leo", include_str!("./many_assigns.leo"));
let ast = leo_parser::parse_ast("./many_assigns.leo", program_string).expect("failed to parse benchmark");
c.bench_function("Ast::many_assigns", |b| b.iter(|| &ast)); c.bench_function("Ast::many_assigns", |b| b.iter(|| &ast));
} }

18
parser/examples/parser.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -15,7 +15,9 @@
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use leo_ast::Ast; use leo_ast::Ast;
use leo_errors::Result; use leo_errors::{emitter::Handler, Result};
use leo_span::symbol::create_session_if_not_set_then;
use std::{env, fs, path::Path}; use std::{env, fs, path::Path};
fn to_leo_tree(filepath: &Path) -> Result<String> { fn to_leo_tree(filepath: &Path) -> Result<String> {
@ -23,12 +25,12 @@ fn to_leo_tree(filepath: &Path) -> Result<String> {
let program_filepath = filepath.to_path_buf(); let program_filepath = filepath.to_path_buf();
let program_string = fs::read_to_string(&program_filepath).expect("failed to open input file"); let program_string = fs::read_to_string(&program_filepath).expect("failed to open input file");
// Parses the Leo file and constructs an ast. // Parses the Leo file constructing an ast which is then serialized.
let ast = leo_parser::parse_ast(filepath.to_str().unwrap(), &program_string)?; create_session_if_not_set_then(|_| {
let handler = Handler::default();
let serialized_leo_ast = Ast::to_json_string(&ast).expect("serialization failed"); let ast = leo_parser::parse_ast(&handler, filepath.to_str().unwrap(), &program_string)?;
Ok(Ast::to_json_string(&ast).expect("serialization failed"))
Ok(serialized_leo_ast) })
} }
fn main() -> Result<()> { fn main() -> Result<()> {

71
parser/src/lib.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -23,19 +23,82 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
pub(crate) mod tokenizer; pub(crate) mod tokenizer;
use leo_input::LeoInputParser;
pub use tokenizer::KEYWORD_TOKENS; pub use tokenizer::KEYWORD_TOKENS;
pub(crate) use tokenizer::*; pub(crate) use tokenizer::*;
pub mod parser; pub mod parser;
pub use parser::*; pub use parser::*;
use leo_ast::Ast; use leo_ast::{Ast, Input};
use leo_errors::emitter::Handler;
use leo_errors::Result; use leo_errors::Result;
#[cfg(test)] #[cfg(test)]
mod test; mod test;
/// Creates a new AST from a given file path and source code text. /// Creates a new AST from a given file path and source code text.
pub fn parse_ast<T: AsRef<str>, Y: AsRef<str>>(path: T, source: Y) -> Result<Ast> { pub fn parse_ast<T: AsRef<str>, Y: AsRef<str>>(handler: &Handler, path: T, source: Y) -> Result<Ast> {
Ok(Ast::new(parser::parse(path.as_ref(), source.as_ref())?)) Ok(Ast::new(parser::parse(handler, path.as_ref(), source.as_ref())?))
}
/// Parses program input from from the input file path and state file path
pub fn parse_program_input<T: AsRef<str>, Y: AsRef<str>, T2: AsRef<str>, Y2: AsRef<str>>(
input_string: T,
input_path: Y,
state_string: T2,
state_path: Y2,
) -> Result<Input> {
let input_syntax_tree = LeoInputParser::parse_file(input_string.as_ref()).map_err(|mut e| {
e.set_path(
input_path.as_ref(),
&input_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
let state_syntax_tree = LeoInputParser::parse_file(state_string.as_ref()).map_err(|mut e| {
e.set_path(
state_path.as_ref(),
&state_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
let mut input = Input::new();
input.parse_input(input_syntax_tree).map_err(|mut e| {
e.set_path(
input_path.as_ref(),
&input_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
input.parse_state(state_syntax_tree).map_err(|mut e| {
e.set_path(
state_path.as_ref(),
&state_string
.as_ref()
.lines()
.map(|x| x.to_string())
.collect::<Vec<String>>()[..],
);
e
})?;
Ok(input)
} }

150
parser/src/parser/context.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -14,42 +14,48 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use std::{borrow::Cow, unimplemented};
use crate::{assert_no_whitespace, tokenizer::*, Token, KEYWORD_TOKENS}; use crate::{assert_no_whitespace, tokenizer::*, Token, KEYWORD_TOKENS};
use leo_ast::*; use leo_ast::*;
use leo_errors::{LeoError, ParserError, Result, Span}; use leo_errors::emitter::Handler;
use leo_errors::{LeoError, ParserError, Result};
use leo_span::{Span, Symbol};
use std::{borrow::Cow, unimplemented};
use tendril::format_tendril; use tendril::format_tendril;
/// Stores a program in tokenized format plus additional context. /// Stores a program in tokenized format plus additional context.
/// May be converted into a [`Program`] AST by parsing all tokens. /// May be converted into a [`Program`] AST by parsing all tokens.
pub struct ParserContext { pub struct ParserContext<'a> {
#[allow(dead_code)]
pub(crate) handler: &'a Handler,
tokens: Vec<SpannedToken>, tokens: Vec<SpannedToken>,
end_span: Span, end_span: Span,
// true if parsing an expression for an if statement -- means circuit inits are not legal // true if parsing an expression for an if statement -- means circuit inits are not legal
pub(crate) fuzzy_struct_state: bool, pub(crate) fuzzy_struct_state: bool,
} }
impl Iterator for ParserContext { impl Iterator for ParserContext<'_> {
type Item = SpannedToken; type Item = SpannedToken;
fn next(&mut self) -> Option<SpannedToken> { fn next(&mut self) -> Option<SpannedToken> {
self.tokens.pop() self.bump()
} }
} }
impl ParserContext { impl<'a> ParserContext<'a> {
/// ///
/// Returns a new [`ParserContext`] type given a vector of tokens. /// Returns a new [`ParserContext`] type given a vector of tokens.
/// ///
pub fn new(mut tokens: Vec<SpannedToken>) -> Self { pub fn new(handler: &'a Handler, mut tokens: Vec<SpannedToken>) -> Self {
tokens.reverse(); tokens.reverse();
// todo: performance optimization here: drain filter // todo: performance optimization here: drain filter
tokens = tokens tokens = tokens
.into_iter() .into_iter()
.filter(|x| !matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_))) .filter(|x| !matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_)))
.collect(); .collect();
ParserContext { Self {
handler,
end_span: tokens end_span: tokens
.iter() .iter()
.find(|x| !x.span.content.trim().is_empty()) .find(|x| !x.span.content.trim().is_empty())
@ -60,6 +66,16 @@ impl ParserContext {
} }
} }
/// Returns the current token if there is one.
pub fn curr(&self) -> Option<&SpannedToken> {
self.tokens.last()
}
/// Emit the error `err`.
pub(crate) fn emit_err(&self, err: ParserError) {
self.handler.emit_err(err.into());
}
/// ///
/// Returns an unexpected end of function [`SyntaxError`]. /// Returns an unexpected end of function [`SyntaxError`].
/// ///
@ -75,12 +91,15 @@ impl ParserContext {
} }
/// ///
/// Returns a reference to the next token or error if it does not exist. /// Returns a reference to the next SpannedToken or error if it does not exist.
/// ///
pub fn peek(&self) -> Result<&SpannedToken> { pub fn peek(&self) -> Result<&SpannedToken> {
self.tokens.last().ok_or_else(|| self.eof()) self.curr().ok_or_else(|| self.eof())
} }
///
/// Returns a reference to the next Token.
///
pub fn peek_token(&self) -> Cow<'_, Token> { pub fn peek_token(&self) -> Cow<'_, Token> {
self.tokens self.tokens
.last() .last()
@ -112,14 +131,19 @@ impl ParserContext {
!self.tokens.is_empty() !self.tokens.is_empty()
} }
/// Advances the current token.
pub fn bump(&mut self) -> Option<SpannedToken> {
self.tokens.pop()
}
/// ///
/// Removes the next token if it exists and returns it, or [None] if /// Removes the next token if it exists and returns it, or [None] if
/// the next token does not exist. /// the next token does not exist.
/// ///
pub fn eat(&mut self, token: Token) -> Option<SpannedToken> { pub fn eat(&mut self, token: Token) -> Option<SpannedToken> {
if let Some(SpannedToken { token: inner, .. }) = self.tokens.last() { if let Some(SpannedToken { token: inner, .. }) = self.curr() {
if &token == inner { if &token == inner {
return self.tokens.pop(); return self.bump();
} }
} }
None None
@ -139,13 +163,12 @@ impl ParserContext {
pub fn eat_identifier(&mut self) -> Option<Identifier> { pub fn eat_identifier(&mut self) -> Option<Identifier> {
if let Some(SpannedToken { if let Some(SpannedToken {
token: Token::Ident(_), .. token: Token::Ident(_), ..
}) = self.tokens.last() }) = self.curr()
{ {
let token = self.tokens.pop().unwrap();
if let SpannedToken { if let SpannedToken {
token: Token::Ident(name), token: Token::Ident(name),
span, span,
} = token } = self.bump().unwrap()
{ {
return Some(Identifier { name, span }); return Some(Identifier { name, span });
} else { } else {
@ -186,6 +209,21 @@ impl ParserContext {
}) })
} }
/// Returns `true` if the next token is Function or if it is a Const followed by Function.
/// Returns `false` otherwise.
pub fn peek_is_function(&self) -> Result<bool> {
let first = &self.peek()?.token;
let next = if self.tokens.len() >= 2 {
&self.peek_next()?.token
} else {
return Ok(false);
};
Ok(matches!(
(first, next),
(Token::Function | Token::At, _) | (Token::Const, Token::Function)
))
}
/// ///
/// Removes the next two tokens if they are a pair of [`GroupCoordinate`] and returns them, /// Removes the next two tokens if they are a pair of [`GroupCoordinate`] and returns them,
/// or [None] if the next token is not a [`GroupCoordinate`]. /// or [None] if the next token is not a [`GroupCoordinate`].
@ -263,13 +301,12 @@ impl ParserContext {
pub fn eat_int(&mut self) -> Option<(PositiveNumber, Span)> { pub fn eat_int(&mut self) -> Option<(PositiveNumber, Span)> {
if let Some(SpannedToken { if let Some(SpannedToken {
token: Token::Int(_), .. token: Token::Int(_), ..
}) = self.tokens.last() }) = self.curr()
{ {
let token = self.tokens.pop().unwrap();
if let SpannedToken { if let SpannedToken {
token: Token::Int(value), token: Token::Int(value),
span, span,
} = token } = self.bump().unwrap()
{ {
return Some((PositiveNumber { value }, span)); return Some((PositiveNumber { value }, span));
} else { } else {
@ -284,9 +321,9 @@ impl ParserContext {
/// the next token does not exist. /// the next token does not exist.
/// ///
pub fn eat_any(&mut self, token: &[Token]) -> Option<SpannedToken> { pub fn eat_any(&mut self, token: &[Token]) -> Option<SpannedToken> {
if let Some(SpannedToken { token: inner, .. }) = self.tokens.last() { if let Some(SpannedToken { token: inner, .. }) = self.curr() {
if token.iter().any(|x| x == inner) { if token.iter().any(|x| x == inner) {
return self.tokens.pop(); return self.bump();
} }
} }
None None
@ -296,9 +333,9 @@ impl ParserContext {
/// Returns the span of the next token if it is equal to the given [`Token`], or error. /// Returns the span of the next token if it is equal to the given [`Token`], or error.
/// ///
pub fn expect(&mut self, token: Token) -> Result<Span> { pub fn expect(&mut self, token: Token) -> Result<Span> {
if let Some(SpannedToken { token: inner, span }) = self.tokens.last() { if let Some(SpannedToken { token: inner, span }) = self.curr() {
if &token == inner { if &token == inner {
Ok(self.tokens.pop().unwrap().span) Ok(self.bump().unwrap().span)
} else { } else {
Err(ParserError::unexpected(inner, token, span).into()) Err(ParserError::unexpected(inner, token, span).into())
} }
@ -311,9 +348,9 @@ impl ParserContext {
/// Returns the span of the next token if it is equal to one of the given [`Token`]s, or error. /// Returns the span of the next token if it is equal to one of the given [`Token`]s, or error.
/// ///
pub fn expect_oneof(&mut self, token: &[Token]) -> Result<SpannedToken> { pub fn expect_oneof(&mut self, token: &[Token]) -> Result<SpannedToken> {
if let Some(SpannedToken { token: inner, span }) = self.tokens.last() { if let Some(SpannedToken { token: inner, span }) = self.curr() {
if token.iter().any(|x| x == inner) { if token.iter().any(|x| x == inner) {
Ok(self.tokens.pop().unwrap()) Ok(self.bump().unwrap())
} else { } else {
return Err(ParserError::unexpected( return Err(ParserError::unexpected(
inner, inner,
@ -334,27 +371,25 @@ impl ParserContext {
pub fn expect_loose_identifier(&mut self) -> Result<Identifier> { pub fn expect_loose_identifier(&mut self) -> Result<Identifier> {
if let Some(token) = self.eat_any(KEYWORD_TOKENS) { if let Some(token) = self.eat_any(KEYWORD_TOKENS) {
return Ok(Identifier { return Ok(Identifier {
name: token.token.to_string().into(), name: token.token.keyword_to_symbol().unwrap(),
span: token.span, span: token.span,
}); });
} }
if let Some((int, span)) = self.eat_int() { if let Some((int, span)) = self.eat_int() {
return Ok(Identifier { name: int.value, span }); let name = Symbol::intern(&int.value);
return Ok(Identifier { name, span });
} }
self.expect_ident() self.expect_ident()
} }
///
/// Returns the [`Identifier`] of the next token if it is an [`Identifier`], or error. /// Returns the [`Identifier`] of the next token if it is an [`Identifier`], or error.
///
pub fn expect_ident(&mut self) -> Result<Identifier> { pub fn expect_ident(&mut self) -> Result<Identifier> {
if let Some(SpannedToken { token: inner, span }) = self.tokens.last() { if let Some(SpannedToken { token: inner, span }) = self.curr() {
if let Token::Ident(_) = inner { if let Token::Ident(_) = inner {
let token = self.tokens.pop().unwrap();
if let SpannedToken { if let SpannedToken {
token: Token::Ident(name), token: Token::Ident(name),
span, span,
} = token } = self.bump().unwrap()
{ {
Ok(Identifier { name, span }) Ok(Identifier { name, span })
} else { } else {
@ -378,4 +413,53 @@ impl ParserContext {
Err(self.eof()) Err(self.eof())
} }
} }
/// Parses a list of `T`s using `inner`
/// The opening and closing delimiters are `bra` and `ket`,
/// and elements in the list are separated by `sep`.
/// When `(list, true)` is returned, `sep` was a terminator.
pub(super) fn parse_list<T>(
&mut self,
open: Token,
close: Token,
sep: Token,
mut inner: impl FnMut(&mut Self) -> Result<Option<T>>,
) -> Result<(Vec<T>, bool, Span)> {
let mut list = Vec::new();
let mut trailing = false;
// Parse opening delimiter.
let open_span = self.expect(open)?;
while self.peek()?.token != close {
// Parse the element. We allow inner parser recovery through the `Option`.
if let Some(elem) = inner(self)? {
list.push(elem);
}
// Parse the separator.
if self.eat(sep.clone()).is_none() {
trailing = false;
break;
}
}
// Parse closing delimiter.
let close_span = self.expect(close)?;
Ok((list, trailing, open_span + close_span))
}
/// Parse a list separated by `,` and delimited by parens.
pub(super) fn parse_paren_comma_list<T>(
&mut self,
f: impl FnMut(&mut Self) -> Result<Option<T>>,
) -> Result<(Vec<T>, bool, Span)> {
self.parse_list(Token::LeftParen, Token::RightParen, Token::Comma, f)
}
/// Returns true if the current token is `(`.
pub(super) fn peek_is_left_par(&self) -> bool {
matches!(self.curr().map(|t| &t.token), Some(Token::LeftParen))
}
} }

282
parser/src/parser/expression.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -14,11 +14,12 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use tendril::format_tendril; use super::*;
use leo_errors::{ParserError, Result}; use leo_errors::{ParserError, Result};
use leo_span::sym;
use super::*; use tendril::format_tendril;
const INT_TYPES: &[Token] = &[ const INT_TYPES: &[Token] = &[
Token::I8, Token::I8,
@ -35,7 +36,7 @@ const INT_TYPES: &[Token] = &[
Token::Group, Token::Group,
]; ];
impl ParserContext { impl ParserContext<'_> {
/// ///
/// Returns an [`Expression`] AST node if the next token is an expression. /// Returns an [`Expression`] AST node if the next token is an expression.
/// Includes circuit init expressions. /// Includes circuit init expressions.
@ -81,44 +82,45 @@ impl ParserContext {
Ok(expr) Ok(expr)
} }
/// /// Constructs a binary expression `left op right`.
/// Returns an [`Expression`] AST node if the next tokens represent fn bin_expr(left: Expression, right: Expression, op: BinaryOperation) -> Expression {
/// a binary or expression. Expression::Binary(BinaryExpression {
/// span: left.span() + right.span(),
/// Otherwise, tries to parse the next token using [`parse_conjunctive_expression`]. op,
/// left: Box::new(left),
pub fn parse_disjunctive_expression(&mut self) -> Result<Expression> { right: Box::new(right),
let mut expr = self.parse_conjunctive_expression()?; })
while self.eat(Token::Or).is_some() { }
let right = self.parse_conjunctive_expression()?;
expr = Expression::Binary(BinaryExpression { /// Parses a left-associative binary expression `<left> token <right>` using `f` for left/right.
span: expr.span() + right.span(), /// The `token` is translated to `op` in the AST.
op: BinaryOperation::Or, fn parse_bin_expr(
left: Box::new(expr), &mut self,
right: Box::new(right), token: Token,
}) op: BinaryOperation,
mut f: impl FnMut(&mut Self) -> Result<Expression>,
) -> Result<Expression> {
let mut expr = f(self)?;
while self.eat(token.clone()).is_some() {
expr = Self::bin_expr(expr, f(self)?, op);
} }
Ok(expr) Ok(expr)
} }
/// Returns an [`Expression`] AST node if the next tokens represent
/// a binary or expression.
/// ///
/// Otherwise, tries to parse the next token using [`parse_conjunctive_expression`].
pub fn parse_disjunctive_expression(&mut self) -> Result<Expression> {
self.parse_bin_expr(Token::Or, BinaryOperation::Or, Self::parse_conjunctive_expression)
}
/// Returns an [`Expression`] AST node if the next tokens represent a /// Returns an [`Expression`] AST node if the next tokens represent a
/// binary and expression. /// binary and expression.
/// ///
/// Otherwise, tries to parse the next token using [`parse_bit_or_expression`]. /// Otherwise, tries to parse the next token using [`parse_equality_expression`].
///
pub fn parse_conjunctive_expression(&mut self) -> Result<Expression> { pub fn parse_conjunctive_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_equality_expression()?; self.parse_bin_expr(Token::And, BinaryOperation::And, Self::parse_equality_expression)
while self.eat(Token::And).is_some() {
let right = self.parse_equality_expression()?;
expr = Expression::Binary(BinaryExpression {
span: expr.span() + right.span(),
op: BinaryOperation::And,
left: Box::new(expr),
right: Box::new(right),
})
}
Ok(expr)
} }
/// ///
@ -169,7 +171,7 @@ impl ParserContext {
/// ///
// pub fn parse_bit_and_expression(&mut self) -> Result<Expression> { // pub fn parse_bit_and_expression(&mut self) -> Result<Expression> {
// let mut expr = self.parse_equality_expression()?; // let mut expr = self.parse_equality_expression()?;
// while self.eat(Token::BitAnd).is_some() { // while self.eat(Token::Ampersand).is_some() {
// let right = self.parse_equality_expression()?; // let right = self.parse_equality_expression()?;
// expr = Expression::Binary(BinaryExpression { // expr = Expression::Binary(BinaryExpression {
// span: expr.span() + right.span(), // span: expr.span() + right.span(),
@ -181,53 +183,41 @@ impl ParserContext {
// Ok(expr) // Ok(expr)
// } // }
///
/// Returns an [`Expression`] AST node if the next tokens represent a /// Returns an [`Expression`] AST node if the next tokens represent a
/// binary equals or not equals expression. /// binary equals or not equals expression.
/// ///
/// Otherwise, tries to parse the next token using [`parse_ordering_expression`]. /// Otherwise, tries to parse the next token using [`parse_ordering_expression`].
///
pub fn parse_equality_expression(&mut self) -> Result<Expression> { pub fn parse_equality_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_ordering_expression()?; let mut expr = self.parse_ordering_expression()?;
if let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Eq, Token::NotEq]) { if let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Eq, Token::NotEq]) {
let right = self.parse_ordering_expression()?; let right = self.parse_ordering_expression()?;
expr = Expression::Binary(BinaryExpression { let op = match op {
span: expr.span() + right.span(), Token::Eq => BinaryOperation::Eq,
op: match op { Token::NotEq => BinaryOperation::Ne,
Token::Eq => BinaryOperation::Eq, _ => unimplemented!(),
Token::NotEq => BinaryOperation::Ne, };
_ => unimplemented!(), expr = Self::bin_expr(expr, right, op);
},
left: Box::new(expr),
right: Box::new(right),
})
} }
Ok(expr) Ok(expr)
} }
///
/// Returns an [`Expression`] AST node if the next tokens represent a /// Returns an [`Expression`] AST node if the next tokens represent a
/// binary relational expression: less than, less than or equals, greater than, greater than or equals. /// binary relational expression: less than, less than or equals, greater than, greater than or equals.
/// ///
/// Otherwise, tries to parse the next token using [`parse_shift_expression`]. /// Otherwise, tries to parse the next token using [`parse_shift_expression`].
///
pub fn parse_ordering_expression(&mut self) -> Result<Expression> { pub fn parse_ordering_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_additive_expression()?; let mut expr = self.parse_additive_expression()?;
while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Lt, Token::LtEq, Token::Gt, Token::GtEq]) while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Lt, Token::LtEq, Token::Gt, Token::GtEq])
{ {
let right = self.parse_additive_expression()?; let right = self.parse_additive_expression()?;
expr = Expression::Binary(BinaryExpression { let op = match op {
span: expr.span() + right.span(), Token::Lt => BinaryOperation::Lt,
op: match op { Token::LtEq => BinaryOperation::Le,
Token::Lt => BinaryOperation::Lt, Token::Gt => BinaryOperation::Gt,
Token::LtEq => BinaryOperation::Le, Token::GtEq => BinaryOperation::Ge,
Token::Gt => BinaryOperation::Gt, _ => unimplemented!(),
Token::GtEq => BinaryOperation::Ge, };
_ => unimplemented!(), expr = Self::bin_expr(expr, right, op);
},
left: Box::new(expr),
right: Box::new(right),
})
} }
Ok(expr) Ok(expr)
} }
@ -257,76 +247,55 @@ impl ParserContext {
// Ok(expr) // Ok(expr)
// } // }
///
/// Returns an [`Expression`] AST node if the next tokens represent a /// Returns an [`Expression`] AST node if the next tokens represent a
/// binary addition or subtraction expression. /// binary addition or subtraction expression.
/// ///
/// Otherwise, tries to parse the next token using [`parse_mul_div_pow_expression`]. /// Otherwise, tries to parse the next token using [`parse_mul_div_pow_expression`].
///
pub fn parse_additive_expression(&mut self) -> Result<Expression> { pub fn parse_additive_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_multiplicative_expression()?; let mut expr = self.parse_multiplicative_expression()?;
while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Add, Token::Minus]) { while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Add, Token::Minus]) {
let right = self.parse_multiplicative_expression()?; let right = self.parse_multiplicative_expression()?;
expr = Expression::Binary(BinaryExpression { let op = match op {
span: expr.span() + right.span(), Token::Add => BinaryOperation::Add,
op: match op { Token::Minus => BinaryOperation::Sub,
Token::Add => BinaryOperation::Add, _ => unimplemented!(),
Token::Minus => BinaryOperation::Sub, };
_ => unimplemented!(), expr = Self::bin_expr(expr, right, op);
},
left: Box::new(expr),
right: Box::new(right),
})
} }
Ok(expr) Ok(expr)
} }
///
/// Returns an [`Expression`] AST node if the next tokens represent a /// Returns an [`Expression`] AST node if the next tokens represent a
/// binary multiplication, division, or modulus expression. /// binary multiplication, division, or modulus expression.
/// ///
/// Otherwise, tries to parse the next token using [`parse_exponential_expression`]. /// Otherwise, tries to parse the next token using [`parse_exponential_expression`].
///
pub fn parse_multiplicative_expression(&mut self) -> Result<Expression> { pub fn parse_multiplicative_expression(&mut self) -> Result<Expression> {
let mut expr = self.parse_exponential_expression()?; let mut expr = self.parse_exponential_expression()?;
while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Mul, Token::Div]) { while let Some(SpannedToken { token: op, .. }) = self.eat_any(&[Token::Mul, Token::Div]) {
let right = self.parse_exponential_expression()?; let right = self.parse_exponential_expression()?;
expr = Expression::Binary(BinaryExpression { let op = match op {
span: expr.span() + right.span(), Token::Mul => BinaryOperation::Mul,
op: match op { Token::Div => BinaryOperation::Div,
Token::Mul => BinaryOperation::Mul, // Token::Mod => BinaryOperation::Mod,
Token::Div => BinaryOperation::Div, _ => unimplemented!(),
// Token::Mod => BinaryOperation::Mod, };
_ => unimplemented!(), expr = Self::bin_expr(expr, right, op);
},
left: Box::new(expr),
right: Box::new(right),
})
} }
Ok(expr) Ok(expr)
} }
///
/// Returns an [`Expression`] AST node if the next tokens represent a /// Returns an [`Expression`] AST node if the next tokens represent a
/// binary exponentiation expression. /// binary exponentiation expression.
/// ///
/// Otherwise, tries to parse the next token using [`parse_cast_expression`]. /// Otherwise, tries to parse the next token using [`parse_cast_expression`].
///
pub fn parse_exponential_expression(&mut self) -> Result<Expression> { pub fn parse_exponential_expression(&mut self) -> Result<Expression> {
let mut exprs = vec![self.parse_cast_expression()?]; let mut expr = self.parse_cast_expression()?;
while self.eat(Token::Exp).is_some() {
exprs.push(self.parse_cast_expression()?); if self.eat(Token::Exp).is_some() {
} let right = self.parse_exponential_expression()?;
let mut expr = exprs.remove(exprs.len() - 1); expr = Self::bin_expr(expr, right, BinaryOperation::Pow);
while !exprs.is_empty() {
let sub_expr = exprs.remove(exprs.len() - 1);
expr = Expression::Binary(BinaryExpression {
span: expr.span() + sub_expr.span(),
op: BinaryOperation::Pow,
left: Box::new(sub_expr),
right: Box::new(expr),
})
} }
Ok(expr) Ok(expr)
} }
@ -401,21 +370,12 @@ impl ParserContext {
/// Otherwise, tries to parse the next token using [`parse_primary_expression`]. /// Otherwise, tries to parse the next token using [`parse_primary_expression`].
/// ///
pub fn parse_postfix_expression(&mut self) -> Result<Expression> { pub fn parse_postfix_expression(&mut self) -> Result<Expression> {
// We don't directly parse named-type's and Identifier's here as
// the ABNF states. Rather the primary expression already
// handle those. The ABNF is more specific for language reasons.
let mut expr = self.parse_primary_expression()?; let mut expr = self.parse_primary_expression()?;
while let Some(token) = self.eat_any(&[ while let Some(token) = self.eat_any(&[Token::LeftSquare, Token::Dot, Token::LeftParen, Token::DoubleColon]) {
Token::LeftSquare,
Token::Dot,
Token::LeftParen,
Token::DoubleColon,
Token::LengthOf,
]) {
match token.token { match token.token {
Token::LengthOf => {
expr = Expression::LengthOf(LengthOfExpression {
span: expr.span().clone(),
inner: Box::new(expr),
})
}
Token::LeftSquare => { Token::LeftSquare => {
if self.eat(Token::DotDot).is_some() { if self.eat(Token::DotDot).is_some() {
let right = if self.peek_token().as_ref() != &Token::RightSquare { let right = if self.peek_token().as_ref() != &Token::RightSquare {
@ -425,12 +385,12 @@ impl ParserContext {
}; };
let end = self.expect(Token::RightSquare)?; let end = self.expect(Token::RightSquare)?;
expr = Expression::ArrayRangeAccess(ArrayRangeAccessExpression { expr = Expression::Access(AccessExpression::ArrayRange(ArrayRangeAccess {
span: expr.span() + &end, span: expr.span() + &end,
array: Box::new(expr), array: Box::new(expr),
left: None, left: None,
right, right,
}); }));
continue; continue;
} }
@ -443,35 +403,35 @@ impl ParserContext {
}; };
let end = self.expect(Token::RightSquare)?; let end = self.expect(Token::RightSquare)?;
expr = Expression::ArrayRangeAccess(ArrayRangeAccessExpression { expr = Expression::Access(AccessExpression::ArrayRange(ArrayRangeAccess {
span: expr.span() + &end, span: expr.span() + &end,
array: Box::new(expr), array: Box::new(expr),
left: Some(Box::new(left)), left: Some(Box::new(left)),
right, right,
}); }));
} else { } else {
let end = self.expect(Token::RightSquare)?; let end = self.expect(Token::RightSquare)?;
expr = Expression::ArrayAccess(ArrayAccessExpression { expr = Expression::Access(AccessExpression::Array(ArrayAccess {
span: expr.span() + &end, span: expr.span() + &end,
array: Box::new(expr), array: Box::new(expr),
index: Box::new(left), index: Box::new(left),
}); }));
} }
} }
Token::Dot => { Token::Dot => {
if let Some(ident) = self.eat_identifier() { if let Some(ident) = self.eat_identifier() {
expr = Expression::CircuitMemberAccess(CircuitMemberAccessExpression { expr = Expression::Access(AccessExpression::Member(MemberAccess {
span: expr.span() + &ident.span, span: expr.span() + &ident.span,
circuit: Box::new(expr), inner: Box::new(expr),
name: ident, name: ident,
type_: None, type_: None,
}); }));
} else if let Some((num, span)) = self.eat_int() { } else if let Some((num, span)) = self.eat_int() {
expr = Expression::TupleAccess(TupleAccessExpression { expr = Expression::Access(AccessExpression::Tuple(TupleAccess {
span: expr.span() + &span, span: expr.span() + &span,
tuple: Box::new(expr), tuple: Box::new(expr),
index: num, index: num,
}); }));
} else { } else {
let next = self.peek()?; let next = self.peek()?;
return Err(ParserError::unexpected_str(&next.token, "int or ident", &next.span).into()); return Err(ParserError::unexpected_str(&next.token, "int or ident", &next.span).into());
@ -481,8 +441,7 @@ impl ParserContext {
let mut arguments = Vec::new(); let mut arguments = Vec::new();
let end_span; let end_span;
loop { loop {
let end = self.eat(Token::RightParen); if let Some(end) = self.eat(Token::RightParen) {
if let Some(end) = end {
end_span = end.span; end_span = end.span;
break; break;
} }
@ -500,11 +459,12 @@ impl ParserContext {
} }
Token::DoubleColon => { Token::DoubleColon => {
let ident = self.expect_ident()?; let ident = self.expect_ident()?;
expr = Expression::CircuitStaticFunctionAccess(CircuitStaticFunctionAccessExpression { expr = Expression::Access(AccessExpression::Static(StaticAccess {
span: expr.span() + &ident.span, span: expr.span() + &ident.span,
circuit: Box::new(expr), inner: Box::new(expr),
type_: None,
name: ident, name: ident,
}); }));
} }
_ => unimplemented!(), _ => unimplemented!(),
} }
@ -526,39 +486,17 @@ impl ParserContext {
}) })
} }
/// /// Returns an [`Expression`] AST node if the next tokens represent a
/// Returns an [`Expression`] AST node if the next tokens represent an
/// circuit initialization expression. /// circuit initialization expression.
///
pub fn parse_circuit_expression(&mut self, identifier: Identifier) -> Result<Expression> { pub fn parse_circuit_expression(&mut self, identifier: Identifier) -> Result<Expression> {
self.expect(Token::LeftCurly)?; let (members, _, span) = self.parse_list(Token::LeftCurly, Token::RightCurly, Token::Comma, |p| {
let mut members = Vec::new(); Ok(Some(CircuitImpliedVariableDefinition {
let end_span; identifier: p.expect_ident()?,
loop { expression: p.eat(Token::Colon).map(|_| p.parse_expression()).transpose()?,
if let Some(end) = self.eat(Token::RightCurly) { }))
end_span = end.span; })?;
break;
}
let name = self.expect_ident()?;
if self.eat(Token::Colon).is_some() {
let expression = self.parse_expression()?;
members.push(CircuitImpliedVariableDefinition {
identifier: name,
expression: Some(expression),
});
} else {
members.push(CircuitImpliedVariableDefinition {
identifier: name.clone(),
expression: None,
});
}
if self.eat(Token::Comma).is_none() {
end_span = self.expect(Token::RightCurly)?;
break;
}
}
Ok(Expression::CircuitInit(CircuitInitExpression { Ok(Expression::CircuitInit(CircuitInitExpression {
span: &identifier.span + &end_span, span: &identifier.span + &span,
name: identifier, name: identifier,
members, members,
})) }))
@ -617,8 +555,8 @@ impl ParserContext {
let first = self.parse_spread_or_expression()?; let first = self.parse_spread_or_expression()?;
if self.eat(Token::Semicolon).is_some() { if self.eat(Token::Semicolon).is_some() {
let dimensions = self let dimensions = self
.parse_array_dimensions()? .parse_array_dimensions()
.ok_or_else(|| ParserError::unable_to_parse_array_dimensions(span))?; .map_err(|_| ParserError::unable_to_parse_array_dimensions(span))?;
let end = self.expect(Token::RightSquare)?; let end = self.expect(Token::RightSquare)?;
let first = match first { let first = match first {
SpreadOrExpression::Spread(first) => { SpreadOrExpression::Spread(first) => {
@ -723,7 +661,7 @@ impl ParserContext {
} }
Token::BigSelf => { Token::BigSelf => {
let ident = Identifier { let ident = Identifier {
name: token.to_string().into(), name: sym::SelfUpper,
span, span,
}; };
if !self.fuzzy_struct_state && self.peek_token().as_ref() == &Token::LeftCurly { if !self.fuzzy_struct_state && self.peek_token().as_ref() == &Token::LeftCurly {
@ -732,17 +670,15 @@ impl ParserContext {
Expression::Identifier(ident) Expression::Identifier(ident)
} }
} }
Token::Input | Token::LittleSelf => { Token::LittleSelf => Expression::Identifier(Identifier {
let ident = Identifier { name: sym::SelfLower,
name: token.to_string().into(), span,
span, }),
}; Token::Input => Expression::Identifier(Identifier { name: sym::input, span }),
Expression::Identifier(ident) t if crate::type_::TYPE_TOKENS.contains(&t) => Expression::Identifier(Identifier {
} name: t.keyword_to_symbol().unwrap(),
// Token::LengthOf => Expression::LengthOf(LengthOfExpression { span,
// span, }),
// inner: Box::new(self.parse_primary_expression()?),
// }),
token => { token => {
return Err(ParserError::unexpected_str(token, "expression", &span).into()); return Err(ParserError::unexpected_str(token, "expression", &span).into());
} }

392
parser/src/parser/file.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -14,15 +14,13 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use tendril::format_tendril; use super::*;
use leo_errors::{ParserError, Result, Span};
use crate::KEYWORD_TOKENS; use crate::KEYWORD_TOKENS;
use super::*; use leo_errors::{ParserError, Result};
use leo_span::{sym, Span};
impl ParserContext { impl ParserContext<'_> {
/// ///
/// Returns a [`Program`] AST if all tokens can be consumed and represent a valid Leo program. /// Returns a [`Program`] AST if all tokens can be consumed and represent a valid Leo program.
/// ///
@ -41,42 +39,38 @@ impl ParserContext {
import_statements.push(self.parse_import_statement()?); import_statements.push(self.parse_import_statement()?);
} }
Token::Circuit => { Token::Circuit => {
self.expect(Token::Circuit)?;
let (id, circuit) = self.parse_circuit()?; let (id, circuit) = self.parse_circuit()?;
circuits.insert(id, circuit); circuits.insert(id, circuit);
} }
Token::Function | Token::At => { Token::Ident(ident) => match *ident {
sym::test => return Err(ParserError::test_function(&token.span).into()),
kw @ (sym::Struct | sym::Class) => {
self.emit_err(ParserError::unexpected(kw, "circuit", &token.span));
self.bump().unwrap();
let (id, circuit) = self.parse_circuit()?;
circuits.insert(id, circuit);
}
_ => return Err(Self::unexpected_item(token).into()),
},
// Const functions share the first token with the global Const.
Token::Const if self.peek_is_function()? => {
let (id, function) = self.parse_function_declaration()?; let (id, function) = self.parse_function_declaration()?;
functions.insert(id, function); functions.insert(id, function);
} }
Token::Ident(ident) if ident.as_ref() == "test" => {
return Err(ParserError::test_function(&token.span).into());
}
Token::Const => { Token::Const => {
let (name, global_const) = self.parse_global_const_declaration()?; let (name, global_const) = self.parse_global_const_declaration()?;
global_consts.insert(name, global_const); global_consts.insert(name, global_const);
} }
Token::Function | Token::At => {
let (id, function) = self.parse_function_declaration()?;
functions.insert(id, function);
}
Token::Type => { Token::Type => {
let (name, alias) = self.parse_type_alias()?; let (name, alias) = self.parse_type_alias()?;
aliases.insert(name, alias); aliases.insert(name, alias);
} }
_ => { _ => return Err(Self::unexpected_item(token).into()),
return Err(ParserError::unexpected(
&token.token,
[
Token::Import,
Token::Circuit,
Token::Function,
Token::Ident("test".into()),
Token::At,
]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
.join(", "),
&token.span,
)
.into());
}
} }
} }
Ok(Program { Ok(Program {
@ -91,58 +85,46 @@ impl ParserContext {
}) })
} }
/// fn unexpected_item(token: &SpannedToken) -> ParserError {
ParserError::unexpected(
&token.token,
[
Token::Import,
Token::Circuit,
Token::Function,
Token::Ident(sym::test),
Token::At,
]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
.join(", "),
&token.span,
)
}
/// Returns an [`Annotation`] AST node if the next tokens represent a supported annotation. /// Returns an [`Annotation`] AST node if the next tokens represent a supported annotation.
///
pub fn parse_annotation(&mut self) -> Result<Annotation> { pub fn parse_annotation(&mut self) -> Result<Annotation> {
let start = self.expect(Token::At)?; let start = self.expect(Token::At)?;
let name = self.expect_ident()?; let name = self.parse_annotation_name()?;
if name.name.as_ref() == "context" {
return Err(ParserError::context_annotation(&name.span).into());
}
assert_no_whitespace(&start, &name.span, &name.name, "@")?; assert_no_whitespace(&start, &name.span, &name.name.as_str(), "@")?;
let end_span; let (end_span, arguments) = if self.peek_is_left_par() {
let arguments = if self.eat(Token::LeftParen).is_some() { let (args, _, span) = self.parse_paren_comma_list(|p| {
let mut args = Vec::new(); Ok(if let Some(ident) = p.eat_identifier() {
let mut comma = false; Some(ident.name)
loop { } else if let Some((int, _)) = p.eat_int() {
if let Some(end) = self.eat(Token::RightParen) { Some(Symbol::intern(&int.value))
if comma {
return Err(ParserError::unexpected(
Token::RightParen,
[Token::Ident("identifier".into()), Token::Int("number".into())]
.iter()
.map(|x| format!("'{}'", x))
.collect::<Vec<_>>()
.join(", "),
&end.span,
)
.into());
}
end_span = end.span;
break;
}
comma = false;
if let Some(ident) = self.eat_identifier() {
args.push(ident.name);
} else if let Some((int, _)) = self.eat_int() {
args.push(int.value);
} else { } else {
let token = self.peek()?; let token = p.expect_any()?;
return Err(ParserError::unexpected_str(&token.token, "ident or int", &token.span).into()); p.emit_err(ParserError::unexpected_str(&token.token, "ident or int", &token.span));
} None
if self.eat(Token::Comma).is_none() && !comma { })
end_span = self.expect(Token::RightParen)?; })?;
break; (span, args)
}
comma = true;
}
args
} else { } else {
end_span = name.span.clone(); (name.span.clone(), Vec::new())
Vec::new()
}; };
Ok(Annotation { Ok(Annotation {
name, name,
@ -151,24 +133,26 @@ impl ParserContext {
}) })
} }
/// /// Parses `foo` in an annotation `@foo . That is, the name of the annotation.
/// Returns a vector of [`PackageAccess`] AST nodes if the next tokens represent package access fn parse_annotation_name(&mut self) -> Result<Identifier> {
/// expressions within an import statement. let mut name = self.expect_ident()?;
///
pub fn parse_package_accesses(&mut self, span: &Span) -> Result<Vec<PackageAccess>> { // Recover `context` instead of `test`.
let mut out = Vec::new(); if name.name == sym::context {
self.expect(Token::LeftParen)?; self.emit_err(ParserError::context_annotation(&name.span));
while self.eat(Token::RightParen).is_none() { name.name = sym::test;
let access = self.parse_package_access()?;
out.push(access);
if self.eat(Token::Comma).is_none() {
self.expect(Token::RightParen)?;
break;
}
} }
Ok(name)
}
/// Returns a vector of [`PackageAccess`] AST nodes if the next tokens represent package access
/// expressions within an import statement.
pub fn parse_package_accesses(&mut self, span: &Span) -> Result<Vec<PackageAccess>> {
let (out, ..) = self.parse_paren_comma_list(|p| p.parse_package_access().map(Some))?;
if out.is_empty() { if out.is_empty() {
return Err(ParserError::invalid_import_list(span).into()); self.emit_err(ParserError::invalid_import_list(span));
} }
Ok(out) Ok(out)
@ -191,7 +175,7 @@ impl ParserContext {
name.span = name.span + span; name.span = name.span + span;
let next = self.expect_ident()?; let next = self.expect_ident()?;
name.span = name.span + next.span; name.span = name.span + next.span;
name.name = format_tendril!("{}-{}", name.name, next.name); name.name = Symbol::intern(&format!("{}-{}", name.name, next.name));
} }
if self.peek_token().as_ref() == &Token::Dot { if self.peek_token().as_ref() == &Token::Dot {
@ -220,9 +204,7 @@ impl ParserContext {
} }
} }
///
/// Returns an [`Identifier`] AST node if the next tokens represent a valid package name. /// Returns an [`Identifier`] AST node if the next tokens represent a valid package name.
///
pub fn parse_package_name(&mut self) -> Result<Identifier> { pub fn parse_package_name(&mut self) -> Result<Identifier> {
// Build the package name, starting with valid characters up to a dash `-` (Token::Minus). // Build the package name, starting with valid characters up to a dash `-` (Token::Minus).
let mut base = self.expect_loose_identifier()?; let mut base = self.expect_loose_identifier()?;
@ -234,22 +216,22 @@ impl ParserContext {
let span = self.expect(Token::Minus)?; let span = self.expect(Token::Minus)?;
base.span = base.span + span; base.span = base.span + span;
let next = self.expect_loose_identifier()?; let next = self.expect_loose_identifier()?;
base.name = format_tendril!("{}-{}", base.name, next.name); base.name = Symbol::intern(&format!("{}-{}", base.name, next.name));
base.span = base.span + next.span; base.span = base.span + next.span;
} }
Token::Int(_) => { Token::Int(_) => {
let (num, span) = self.eat_int().unwrap(); let (num, span) = self.eat_int().unwrap();
base.name = format_tendril!("{}{}", base.name, num.value); base.name = Symbol::intern(&format!("{}{}", base.name, num.value));
base.span = base.span + span; base.span = base.span + span;
} }
Token::Ident(_) => { Token::Ident(_) => {
let next = self.expect_ident()?; let next = self.expect_ident()?;
base.name = format_tendril!("{}{}", base.name, next.name); base.name = Symbol::intern(&format!("{}{}", base.name, next.name));
base.span = base.span + next.span; base.span = base.span + next.span;
} }
x if KEYWORD_TOKENS.contains(x) => { x if KEYWORD_TOKENS.contains(x) => {
let next = self.expect_loose_identifier()?; let next = self.expect_loose_identifier()?;
base.name = format_tendril!("{}{}", base.name, next.name); base.name = Symbol::intern(&format!("{}{}", base.name, next.name));
base.span = base.span + next.span; base.span = base.span + next.span;
} }
_ => break, _ => break,
@ -257,17 +239,18 @@ impl ParserContext {
} }
// Return an error if the package name contains a keyword. // Return an error if the package name contains a keyword.
if let Some(token) = KEYWORD_TOKENS.iter().find(|x| x.to_string() == base.name.as_ref()) { if let Some(token) = KEYWORD_TOKENS.iter().find(|x| x.keyword_to_symbol() == Some(base.name)) {
return Err(ParserError::unexpected_str(token, "package name", &base.span).into()); self.emit_err(ParserError::unexpected_str(token, "package name", &base.span));
} }
// Return an error if the package name contains invalid characters. // Return an error if the package name contains invalid characters.
if !base if !base
.name .name
.as_str()
.chars() .chars()
.all(|x| x.is_ascii_lowercase() || x.is_ascii_digit() || x == '-' || x == '_') .all(|x| x.is_ascii_lowercase() || x.is_ascii_digit() || x == '-' || x == '_')
{ {
return Err(ParserError::invalid_package_name(&base.span).into()); self.emit_err(ParserError::invalid_package_name(&base.span));
} }
// Return the package name. // Return the package name.
@ -281,7 +264,7 @@ impl ParserContext {
pub fn parse_package_path(&mut self) -> Result<PackageOrPackages> { pub fn parse_package_path(&mut self) -> Result<PackageOrPackages> {
let package_name = self.parse_package_name()?; let package_name = self.parse_package_name()?;
self.expect(Token::Dot)?; self.expect(Token::Dot)?;
if self.peek()?.token == Token::LeftParen { if self.peek_is_left_par() {
let accesses = self.parse_package_accesses(&package_name.span)?; let accesses = self.parse_package_accesses(&package_name.span)?;
Ok(PackageOrPackages::Packages(Packages { Ok(PackageOrPackages::Packages(Packages {
span: &package_name.span + accesses.last().map(|x| x.span()).unwrap_or(&package_name.span), span: &package_name.span + accesses.last().map(|x| x.span()).unwrap_or(&package_name.span),
@ -311,83 +294,112 @@ impl ParserContext {
}) })
} }
///
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member variable /// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member variable
/// or circuit member function. /// or circuit member function.
///
pub fn parse_circuit_declaration(&mut self) -> Result<Vec<CircuitMember>> { pub fn parse_circuit_declaration(&mut self) -> Result<Vec<CircuitMember>> {
let mut members = Vec::new(); let mut members = Vec::new();
let peeked = &self.peek()?.token;
let mut last_variable = peeked == &Token::Function || peeked == &Token::At;
let (mut semi_colons, mut commas) = (false, false); let (mut semi_colons, mut commas) = (false, false);
while self.eat(Token::RightCurly).is_none() { while self.eat(Token::RightCurly).is_none() {
if !last_variable { members.push(if self.peek_is_function()? {
let (variable, last) = self.parse_member_variable_declaration()?; // function
self.parse_member_function_declaration()?
members.push(variable); } else if *self.peek_token() == Token::Static {
// static const
let peeked = &self.peek()?; self.parse_const_member_variable_declaration()?
if peeked.token == Token::Semicolon {
if commas {
return Err(ParserError::mixed_commas_and_semicolons(&peeked.span).into());
}
semi_colons = true;
self.expect(Token::Semicolon)?;
} else {
if semi_colons {
return Err(ParserError::mixed_commas_and_semicolons(&peeked.span).into());
}
commas = true;
self.eat(Token::Comma);
}
if last {
last_variable = last;
}
} else { } else {
let function = self.parse_member_function_declaration()?; // variable
members.push(function); let variable = self.parse_member_variable_declaration()?;
}
if let Some(semi) = self.eat(Token::Semicolon) {
if commas {
self.emit_err(ParserError::mixed_commas_and_semicolons(&semi.span));
}
semi_colons = true;
}
if let Some(comma) = self.eat(Token::Comma) {
if semi_colons {
self.emit_err(ParserError::mixed_commas_and_semicolons(&comma.span));
}
commas = true;
}
variable
});
} }
self.ban_mixed_member_order(&members);
Ok(members) Ok(members)
} }
/// Emits errors if order isn't `consts variables functions`.
fn ban_mixed_member_order(&self, members: &[CircuitMember]) {
let mut had_var = false;
let mut had_fun = false;
for member in members {
match member {
CircuitMember::CircuitConst(id, _, e) if had_var => {
self.emit_err(ParserError::member_const_after_var(&(id.span() + e.span())));
}
CircuitMember::CircuitConst(id, _, e) if had_fun => {
self.emit_err(ParserError::member_const_after_fun(&(id.span() + e.span())));
}
CircuitMember::CircuitVariable(id, _) if had_fun => {
self.emit_err(ParserError::member_var_after_fun(id.span()));
}
CircuitMember::CircuitConst(..) => {}
CircuitMember::CircuitVariable(..) => had_var = true,
CircuitMember::CircuitFunction(..) => had_fun = true,
}
}
}
/// Parses `IDENT: TYPE`.
fn parse_typed_field_name(&mut self) -> Result<(Identifier, Type)> {
let name = self.expect_ident()?;
self.expect(Token::Colon)?;
let type_ = self.parse_type()?.0;
Ok((name, type_))
}
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member static constant.
pub fn parse_const_member_variable_declaration(&mut self) -> Result<CircuitMember> {
self.expect(Token::Static)?;
self.expect(Token::Const)?;
// `IDENT: TYPE = EXPR`:
let (name, type_) = self.parse_typed_field_name()?;
self.expect(Token::Assign)?;
let literal = self.parse_primary_expression()?;
self.expect(Token::Semicolon)?;
Ok(CircuitMember::CircuitConst(name, type_, literal))
}
/// ///
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member variable. /// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member variable.
/// ///
pub fn parse_member_variable_declaration(&mut self) -> Result<(CircuitMember, bool)> { pub fn parse_member_variable_declaration(&mut self) -> Result<CircuitMember> {
let name = self.expect_ident()?; let (name, type_) = self.parse_typed_field_name()?;
self.expect(Token::Colon)?;
let type_ = self.parse_type()?.0;
let peeked = &self.peek()?.token; Ok(CircuitMember::CircuitVariable(name, type_))
if peeked == &Token::Function || peeked == &Token::At || peeked == &Token::RightCurly {
return Ok((CircuitMember::CircuitVariable(name, type_), true));
} else if peeked == &Token::Comma || peeked == &Token::Semicolon {
let peeked = &self.peek_next()?.token;
if peeked == &Token::Function || peeked == &Token::At || peeked == &Token::RightCurly {
return Ok((CircuitMember::CircuitVariable(name, type_), true));
}
}
Ok((CircuitMember::CircuitVariable(name, type_), false))
} }
///
/// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member function. /// Returns a [`CircuitMember`] AST node if the next tokens represent a circuit member function.
///
pub fn parse_member_function_declaration(&mut self) -> Result<CircuitMember> { pub fn parse_member_function_declaration(&mut self) -> Result<CircuitMember> {
let peeked = self.peek()?.clone(); let peeked = self.peek()?.clone();
if peeked.token == Token::Function || peeked.token == Token::At { if self.peek_is_function()? {
let function = self.parse_function_declaration()?; let function = self.parse_function_declaration()?;
Ok(CircuitMember::CircuitFunction(function.1)) Ok(CircuitMember::CircuitFunction(Box::new(function.1)))
} else { } else {
return Err(ParserError::unexpected( return Err(ParserError::unexpected(
&peeked.token, &peeked.token,
[Token::Function, Token::At] [Token::Function, Token::At, Token::Const]
.iter() .iter()
.map(|x| format!("'{}'", x)) .map(|x| format!("'{}'", x))
.collect::<Vec<_>>() .collect::<Vec<_>>()
@ -403,8 +415,18 @@ impl ParserContext {
/// circuit name and definition statement. /// circuit name and definition statement.
/// ///
pub fn parse_circuit(&mut self) -> Result<(Identifier, Circuit)> { pub fn parse_circuit(&mut self) -> Result<(Identifier, Circuit)> {
self.expect(Token::Circuit)?; let name = if let Some(ident) = self.eat_identifier() {
let name = self.expect_ident()?; ident
} else if let Some(scalar_type) = self.eat_any(crate::type_::TYPE_TOKENS) {
Identifier {
name: scalar_type.token.keyword_to_symbol().unwrap(),
span: scalar_type.span,
}
} else {
let next = self.peek()?;
return Err(ParserError::unexpected_str(&next.token, "ident", &next.span).into());
};
self.expect(Token::LeftCurly)?; self.expect(Token::LeftCurly)?;
let members = self.parse_circuit_declaration()?; let members = self.parse_circuit_declaration()?;
@ -412,7 +434,6 @@ impl ParserContext {
name.clone(), name.clone(),
Circuit { Circuit {
circuit_name: name, circuit_name: name,
core_mapping: std::cell::RefCell::new(None),
members, members,
}, },
)) ))
@ -424,24 +445,26 @@ impl ParserContext {
pub fn parse_function_parameters(&mut self) -> Result<FunctionInput> { pub fn parse_function_parameters(&mut self) -> Result<FunctionInput> {
let const_ = self.eat(Token::Const); let const_ = self.eat(Token::Const);
let mutable = self.eat(Token::Mut); let mutable = self.eat(Token::Mut);
let reference = self.eat(Token::Ampersand);
let mut name = if let Some(token) = self.eat(Token::LittleSelf) { let mut name = if let Some(token) = self.eat(Token::LittleSelf) {
Identifier { Identifier {
name: token.token.to_string().into(), name: sym::SelfLower,
span: token.span, span: token.span,
} }
} else { } else {
self.expect_ident()? self.expect_ident()?
}; };
if name.name.as_ref() == "self" { if name.name == sym::SelfLower {
if let Some(mutable) = &mutable { if let Some(mutable) = &mutable {
// Handle `mut self`. self.emit_err(ParserError::mut_self_parameter(&(&mutable.span + &name.span)));
name.span = &mutable.span + &name.span; return Ok(Self::build_ref_self(name, mutable));
name.name = "mut self".to_string().into(); } else if let Some(reference) = &reference {
return Ok(FunctionInput::MutSelfKeyword(MutSelfKeyword { identifier: name })); // Handle `&self`.
return Ok(Self::build_ref_self(name, reference));
} else if let Some(const_) = &const_ { } else if let Some(const_) = &const_ {
// Handle `const self`. // Handle `const self`.
name.span = &const_.span + &name.span; name.span = &const_.span + &name.span;
name.name = "const self".to_string().into(); name.name = Symbol::intern("const self");
return Ok(FunctionInput::ConstSelfKeyword(ConstSelfKeyword { identifier: name })); return Ok(FunctionInput::ConstSelfKeyword(ConstSelfKeyword { identifier: name }));
} }
// Handle `self`. // Handle `self`.
@ -449,7 +472,7 @@ impl ParserContext {
} }
if let Some(mutable) = &mutable { if let Some(mutable) = &mutable {
return Err(ParserError::mut_function_input(&(&mutable.span + &name.span)).into()); self.emit_err(ParserError::mut_function_input(&(&mutable.span + &name.span)));
} }
self.expect(Token::Colon)?; self.expect(Token::Colon)?;
@ -463,42 +486,55 @@ impl ParserContext {
})) }))
} }
/// /// Builds a function parameter `&self`.
fn build_ref_self(mut name: Identifier, reference: &SpannedToken) -> FunctionInput {
name.span = &reference.span + &name.span;
// FIXME(Centril): This should be *two* tokens, NOT one!
name.name = Symbol::intern("&self");
FunctionInput::RefSelfKeyword(RefSelfKeyword { identifier: name })
}
/// Returns an [`(Identifier, Function)`] AST node if the next tokens represent a function name /// Returns an [`(Identifier, Function)`] AST node if the next tokens represent a function name
/// and function definition. /// and function definition.
///
pub fn parse_function_declaration(&mut self) -> Result<(Identifier, Function)> { pub fn parse_function_declaration(&mut self) -> Result<(Identifier, Function)> {
let mut annotations = Vec::new(); // Parse any annotations.
let mut annotations = IndexMap::new();
while self.peek_token().as_ref() == &Token::At { while self.peek_token().as_ref() == &Token::At {
annotations.push(self.parse_annotation()?); let annotation = self.parse_annotation()?;
annotations.insert(annotation.name.name, annotation);
} }
// Parse optional const modifier.
let const_ = self.eat(Token::Const).is_some();
// Parse `function IDENT`.
let start = self.expect(Token::Function)?; let start = self.expect(Token::Function)?;
let name = self.expect_ident()?; let name = self.expect_ident()?;
self.expect(Token::LeftParen)?;
let mut inputs = Vec::new(); // Parse parameters.
while self.eat(Token::RightParen).is_none() { let (inputs, ..) = self.parse_paren_comma_list(|p| p.parse_function_parameters().map(Some))?;
let input = self.parse_function_parameters()?;
inputs.push(input); // Parse return type.
if self.eat(Token::Comma).is_none() {
self.expect(Token::RightParen)?;
break;
}
}
let output = if self.eat(Token::Arrow).is_some() { let output = if self.eat(Token::Arrow).is_some() {
Some(self.parse_type()?.0) Some(self.parse_type()?.0)
} else { } else {
None None
}; };
// Parse the function body.
let block = self.parse_block()?; let block = self.parse_block()?;
Ok(( Ok((
name.clone(), name.clone(),
Function { Function {
annotations, annotations,
identifier: name, identifier: name,
input: inputs, input: inputs,
const_,
output, output,
span: start + block.span.clone(), span: start + block.span.clone(),
block, block,
core_mapping: <_>::default(),
}, },
)) ))
} }
@ -513,7 +549,7 @@ impl ParserContext {
.variable_names .variable_names
.iter() .iter()
.map(|variable_name| variable_name.identifier.clone()) .map(|variable_name| variable_name.identifier.clone())
.collect::<Vec<Identifier>>(); .collect();
Ok((variable_names, statement)) Ok((variable_names, statement))
} }

23
parser/src/parser/mod.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -19,6 +19,16 @@
//! This module contains the [`parse()`] method which calls the underlying [`tokenize()`] //! This module contains the [`parse()`] method which calls the underlying [`tokenize()`]
//! method to create a new program ast. //! method to create a new program ast.
use crate::{tokenizer::*, Token};
use leo_ast::*;
use leo_errors::emitter::Handler;
use leo_errors::{ParserError, Result};
use leo_span::{Span, Symbol};
use indexmap::IndexMap;
use std::unimplemented;
mod context; mod context;
pub use context::*; pub use context::*;
@ -27,13 +37,6 @@ pub mod file;
pub mod statement; pub mod statement;
pub mod type_; pub mod type_;
use std::unimplemented;
use crate::{tokenizer::*, Token};
use indexmap::IndexMap;
use leo_ast::*;
use leo_errors::{ParserError, Result, Span};
pub(crate) fn assert_no_whitespace(left_span: &Span, right_span: &Span, left: &str, right: &str) -> Result<()> { pub(crate) fn assert_no_whitespace(left_span: &Span, right_span: &Span, left: &str, right: &str) -> Result<()> {
if left_span.col_stop != right_span.col_start { if left_span.col_stop != right_span.col_start {
let mut error_span = left_span + right_span; let mut error_span = left_span + right_span;
@ -46,8 +49,8 @@ pub(crate) fn assert_no_whitespace(left_span: &Span, right_span: &Span, left: &s
} }
/// Creates a new program from a given file path and source code text. /// Creates a new program from a given file path and source code text.
pub fn parse(path: &str, source: &str) -> Result<Program> { pub fn parse(handler: &Handler, path: &str, source: &str) -> Result<Program> {
let mut tokens = ParserContext::new(crate::tokenize(path, source.into())?); let mut tokens = ParserContext::new(handler, crate::tokenize(path, source.into())?);
tokens.parse_program() tokens.parse_program()
} }

163
parser/src/parser/statement.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -17,6 +17,7 @@
use super::*; use super::*;
use leo_errors::{ParserError, Result}; use leo_errors::{ParserError, Result};
use leo_span::sym;
const ASSIGN_TOKENS: &[Token] = &[ const ASSIGN_TOKENS: &[Token] = &[
Token::Assign, Token::Assign,
@ -36,7 +37,7 @@ const ASSIGN_TOKENS: &[Token] = &[
// Token::AndEq, // Token::AndEq,
]; ];
impl ParserContext { impl ParserContext<'_> {
/// ///
/// Returns an [`Identifier`] AST node if the given [`Expression`] AST node evaluates to an /// Returns an [`Identifier`] AST node if the given [`Expression`] AST node evaluates to an
/// identifier access. The access is stored in the given accesses. /// identifier access. The access is stored in the given accesses.
@ -44,25 +45,29 @@ impl ParserContext {
pub fn construct_assignee_access(expr: Expression, accesses: &mut Vec<AssigneeAccess>) -> Result<Identifier> { pub fn construct_assignee_access(expr: Expression, accesses: &mut Vec<AssigneeAccess>) -> Result<Identifier> {
let identifier; let identifier;
match expr { match expr {
Expression::CircuitMemberAccess(expr) => { Expression::Access(access) => match access {
identifier = Self::construct_assignee_access(*expr.circuit, accesses)?; AccessExpression::Member(expr) => {
accesses.push(AssigneeAccess::Member(expr.name)); identifier = Self::construct_assignee_access(*expr.inner, accesses)?;
} accesses.push(AssigneeAccess::Member(expr.name));
Expression::TupleAccess(expr) => { }
identifier = Self::construct_assignee_access(*expr.tuple, accesses)?; AccessExpression::Tuple(expr) => {
accesses.push(AssigneeAccess::Tuple(expr.index, expr.span)); identifier = Self::construct_assignee_access(*expr.tuple, accesses)?;
} accesses.push(AssigneeAccess::Tuple(expr.index, expr.span));
Expression::ArrayRangeAccess(expr) => { }
identifier = Self::construct_assignee_access(*expr.array, accesses)?; AccessExpression::ArrayRange(expr) => {
accesses.push(AssigneeAccess::ArrayRange( identifier = Self::construct_assignee_access(*expr.array, accesses)?;
expr.left.map(|x| *x), accesses.push(AssigneeAccess::ArrayRange(
expr.right.map(|x| *x), expr.left.map(|x| *x),
)); expr.right.map(|x| *x),
} ));
Expression::ArrayAccess(expr) => { }
identifier = Self::construct_assignee_access(*expr.array, accesses)?; AccessExpression::Array(expr) => {
accesses.push(AssigneeAccess::ArrayIndex(*expr.index)); identifier = Self::construct_assignee_access(*expr.array, accesses)?;
} accesses.push(AssigneeAccess::ArrayIndex(*expr.index));
}
_ => return Err(ParserError::invalid_assignment_target(access.span()).into()),
},
Expression::Identifier(id) => identifier = id, Expression::Identifier(id) => identifier = id,
_ => return Err(ParserError::invalid_assignment_target(expr.span()).into()), _ => return Err(ParserError::invalid_assignment_target(expr.span()).into()),
} }
@ -177,9 +182,7 @@ impl ParserContext {
}) })
} }
///
/// Returns a [`ConditionalStatement`] AST node if the next tokens represent a conditional statement. /// Returns a [`ConditionalStatement`] AST node if the next tokens represent a conditional statement.
///
pub fn parse_conditional_statement(&mut self) -> Result<ConditionalStatement> { pub fn parse_conditional_statement(&mut self) -> Result<ConditionalStatement> {
let start = self.expect(Token::If)?; let start = self.expect(Token::If)?;
self.fuzzy_struct_state = true; self.fuzzy_struct_state = true;
@ -188,12 +191,10 @@ impl ParserContext {
let body = self.parse_block()?; let body = self.parse_block()?;
let next = if self.eat(Token::Else).is_some() { let next = if self.eat(Token::Else).is_some() {
let s = self.parse_statement()?; let s = self.parse_statement()?;
match s { if !matches!(s, Statement::Block(_) | Statement::Conditional(_)) {
Statement::Block(_) | Statement::Conditional(_) => Some(Box::new(s)), self.emit_err(ParserError::unexpected_statement(&s, "Block or Conditional", s.span()));
s => {
return Err(ParserError::unexpected_statement(&s, "Block or Conditional", s.span()).into());
}
} }
Some(Box::new(s))
} else { } else {
None None
}; };
@ -206,19 +207,20 @@ impl ParserContext {
}) })
} }
///
/// Returns an [`IterationStatement`] AST node if the next tokens represent an iteration statement. /// Returns an [`IterationStatement`] AST node if the next tokens represent an iteration statement.
///
pub fn parse_loop_statement(&mut self) -> Result<IterationStatement> { pub fn parse_loop_statement(&mut self) -> Result<IterationStatement> {
let start_span = self.expect(Token::For)?; let start_span = self.expect(Token::For)?;
let ident = self.expect_ident()?; let ident = self.expect_ident()?;
self.expect(Token::In)?; self.expect(Token::In)?;
// Parse iteration range.
let start = self.parse_expression()?; let start = self.parse_expression()?;
self.expect(Token::DotDot)?; self.expect(Token::DotDot)?;
let inclusive = self.eat(Token::Assign).is_some(); let inclusive = self.eat(Token::Assign).is_some();
self.fuzzy_struct_state = true; self.fuzzy_struct_state = true;
let stop = self.parse_conditional_expression()?; let stop = self.parse_conditional_expression()?;
self.fuzzy_struct_state = false; self.fuzzy_struct_state = false;
let block = self.parse_block()?; let block = self.parse_block()?;
Ok(IterationStatement { Ok(IterationStatement {
@ -231,59 +233,56 @@ impl ParserContext {
}) })
} }
///
/// Returns a [`ConsoleArgs`] AST node if the next tokens represent a formatted string. /// Returns a [`ConsoleArgs`] AST node if the next tokens represent a formatted string.
///
pub fn parse_console_args(&mut self) -> Result<ConsoleArgs> { pub fn parse_console_args(&mut self) -> Result<ConsoleArgs> {
let start_span; let mut string = None;
let string = match self.expect_any()? { let (parameters, _, span) = self.parse_paren_comma_list(|p| {
SpannedToken { if string.is_none() {
token: Token::StringLit(chars), let SpannedToken { token, span } = p.expect_any()?;
span, string = Some(match token {
} => { Token::StringLit(chars) => chars,
start_span = span; _ => {
chars p.emit_err(ParserError::unexpected_str(token, "formatted string", &span));
Vec::new()
}
});
Ok(None)
} else {
p.parse_expression().map(Some)
} }
SpannedToken { token, span } => { })?;
return Err(ParserError::unexpected_str(token, "formatted string", &span).into());
}
};
// let parts = FormatStringPart::from_string(string);
let mut parameters = Vec::new();
while self.eat(Token::Comma).is_some() {
let param = self.parse_expression()?;
parameters.push(param);
}
Ok(ConsoleArgs { Ok(ConsoleArgs {
string, string: string.unwrap_or_default(),
span: &start_span + parameters.last().map(|x| x.span()).unwrap_or(&start_span), span,
parameters, parameters,
}) })
} }
///
/// Returns a [`ConsoleStatement`] AST node if the next tokens represent a console statement. /// Returns a [`ConsoleStatement`] AST node if the next tokens represent a console statement.
///
pub fn parse_console_statement(&mut self) -> Result<ConsoleStatement> { pub fn parse_console_statement(&mut self) -> Result<ConsoleStatement> {
let keyword = self.expect(Token::Console)?; let keyword = self.expect(Token::Console)?;
self.expect(Token::Dot)?; self.expect(Token::Dot)?;
let function = self.expect_ident()?; let function = self.expect_ident()?;
self.expect(Token::LeftParen)?; let function = match function.name {
let function = match &*function.name { sym::assert => {
"assert" => { self.expect(Token::LeftParen)?;
let expr = self.parse_expression()?; let expr = self.parse_expression()?;
self.expect(Token::RightParen)?;
ConsoleFunction::Assert(expr) ConsoleFunction::Assert(expr)
} }
"error" => ConsoleFunction::Error(self.parse_console_args()?), sym::error => ConsoleFunction::Error(self.parse_console_args()?),
"log" => ConsoleFunction::Log(self.parse_console_args()?), sym::log => ConsoleFunction::Log(self.parse_console_args()?),
x => { x => {
return Err(ParserError::unexpected_ident(x, &["assert", "error", "log"], &function.span).into()); // Not sure what it is, assume it's `log`.
self.emit_err(ParserError::unexpected_ident(
x,
&["assert", "error", "log"],
&function.span,
));
ConsoleFunction::Log(self.parse_console_args()?)
} }
}; };
self.expect(Token::RightParen)?;
self.expect(Token::Semicolon)?; self.expect(Token::Semicolon)?;
Ok(ConsoleStatement { Ok(ConsoleStatement {
@ -292,14 +291,13 @@ impl ParserContext {
}) })
} }
///
/// Returns a [`VariableName`] AST node if the next tokens represent a variable name with /// Returns a [`VariableName`] AST node if the next tokens represent a variable name with
/// valid keywords. /// valid keywords.
///
pub fn parse_variable_name(&mut self, span: &SpannedToken) -> Result<VariableName> { pub fn parse_variable_name(&mut self, span: &SpannedToken) -> Result<VariableName> {
let mutable = self.eat(Token::Mut); let mutable = self.eat(Token::Mut);
if let Some(mutable) = &mutable { if let Some(mutable) = &mutable {
return Err(ParserError::let_mut_statement(&(&mutable.span + &span.span)).into()); self.emit_err(ParserError::let_mut_statement(&(&mutable.span + &span.span)));
} }
let name = self.expect_ident()?; let name = self.expect_ident()?;
@ -310,35 +308,24 @@ impl ParserContext {
}) })
} }
///
/// Returns a [`DefinitionStatement`] AST node if the next tokens represent a definition statement. /// Returns a [`DefinitionStatement`] AST node if the next tokens represent a definition statement.
///
pub fn parse_definition_statement(&mut self) -> Result<DefinitionStatement> { pub fn parse_definition_statement(&mut self) -> Result<DefinitionStatement> {
let declare = self.expect_oneof(&[Token::Let, Token::Const])?; let declare = self.expect_oneof(&[Token::Let, Token::Const])?;
let mut variable_names = Vec::new();
let next = self.eat(Token::LeftParen); // Parse variable names.
variable_names.push(self.parse_variable_name(&declare)?); let variable_names = if self.peek_is_left_par() {
if next.is_some() { self.parse_paren_comma_list(|p| p.parse_variable_name(&declare).map(Some))
let mut eaten_ending = false; .map(|(vars, ..)| vars)?
while self.eat(Token::Comma).is_some() {
if self.eat(Token::RightParen).is_some() {
eaten_ending = true;
break;
}
variable_names.push(self.parse_variable_name(&declare)?);
}
if !eaten_ending {
self.expect(Token::RightParen)?;
}
}
let type_ = if self.eat(Token::Colon).is_some() {
Some(self.parse_type()?.0)
} else { } else {
None vec![self.parse_variable_name(&declare)?]
}; };
// Parse an optional type ascription.
let type_ = self
.eat(Token::Colon)
.map(|_| self.parse_type().map(|t| t.0))
.transpose()?;
self.expect(Token::Assign)?; self.expect(Token::Assign)?;
let expr = self.parse_expression()?; let expr = self.parse_expression()?;
self.expect(Token::Semicolon)?; self.expect(Token::Semicolon)?;

77
parser/src/parser/type_.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -17,7 +17,7 @@
use super::*; use super::*;
use leo_errors::{ParserError, Result}; use leo_errors::{ParserError, Result};
const TYPE_TOKENS: &[Token] = &[ pub(crate) const TYPE_TOKENS: &[Token] = &[
Token::I8, Token::I8,
Token::I16, Token::I16,
Token::I32, Token::I32,
@ -35,7 +35,7 @@ const TYPE_TOKENS: &[Token] = &[
Token::Char, Token::Char,
]; ];
impl ParserContext { impl ParserContext<'_> {
/// ///
/// Returns a [`IntegerType`] AST node if the given token is a supported integer type, or [`None`]. /// Returns a [`IntegerType`] AST node if the given token is a supported integer type, or [`None`].
/// ///
@ -55,58 +55,51 @@ impl ParserContext {
}) })
} }
///
/// Returns an [`ArrayDimensions`] AST node if the next tokens represent dimensions for an array type. /// Returns an [`ArrayDimensions`] AST node if the next tokens represent dimensions for an array type.
/// pub fn parse_array_dimensions(&mut self) -> Result<ArrayDimensions> {
pub fn parse_array_dimensions(&mut self) -> Result<Option<ArrayDimensions>> { Ok(if let Some(dim) = self.parse_array_dimension() {
Ok(if let Some((int, _)) = self.eat_int() { dim
Some(ArrayDimensions(vec![int]))
} else if self.eat(Token::Underscore).is_some() {
None
} else { } else {
self.expect(Token::LeftParen)?; let mut had_item_err = false;
let mut dimensions = Vec::new(); let (dims, _, span) = self.parse_paren_comma_list(|p| {
loop { Ok(if let Some(dim) = p.parse_array_dimension() {
if let Some((int, _)) = self.eat_int() { Some(dim)
dimensions.push(int);
} else { } else {
let token = self.peek()?; let token = p.expect_any()?;
return Err(ParserError::unexpected_str(&token.token, "int", &token.span).into()); p.emit_err(ParserError::unexpected_str(&token.token, "int", &token.span));
} had_item_err = true;
if self.eat(Token::Comma).is_none() { None
break; })
} })?;
if dims.is_empty() && !had_item_err {
self.emit_err(ParserError::array_tuple_dimensions_empty(&span));
} }
self.expect(Token::RightParen)?; ArrayDimensions::Multi(dims)
Some(ArrayDimensions(dimensions))
}) })
} }
/// /// Parses a basic array dimension, i.e., an integer or `_`.
/// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type. Also fn parse_array_dimension(&mut self) -> Option<ArrayDimensions> {
/// returns the span of the parsed token. if let Some((int, _)) = self.eat_int() {
/// Some(ArrayDimensions::Number(int))
} else if self.eat(Token::Underscore).is_some() {
Some(ArrayDimensions::Unspecified)
} else {
None
}
}
/// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type.
/// Also returns the span of the parsed token.
pub fn parse_type(&mut self) -> Result<(Type, Span)> { pub fn parse_type(&mut self) -> Result<(Type, Span)> {
Ok(if let Some(token) = self.eat(Token::BigSelf) { Ok(if let Some(token) = self.eat(Token::BigSelf) {
(Type::SelfType, token.span) (Type::SelfType, token.span)
} else if let Some(ident) = self.eat_identifier() { } else if let Some(ident) = self.eat_identifier() {
let span = ident.span.clone(); let span = ident.span.clone();
(Type::Identifier(ident), span) (Type::Identifier(ident), span)
} else if let Some(token) = self.eat(Token::LeftParen) { } else if self.peek_is_left_par() {
let mut types = Vec::new(); let (types, _, span) = self.parse_paren_comma_list(|p| p.parse_type().map(|t| Some(t.0)))?;
let end_span; (Type::Tuple(types), span)
loop {
if let Some(end) = self.eat(Token::RightParen) {
end_span = end.span;
break;
}
types.push(self.parse_type()?.0);
if self.eat(Token::Comma).is_none() {
end_span = self.expect(Token::RightParen)?;
break;
}
}
(Type::Tuple(types), token.span + end_span)
} else if let Some(token) = self.eat(Token::LeftSquare) { } else if let Some(token) = self.eat(Token::LeftSquare) {
let (inner, _) = self.parse_type()?; let (inner, _) = self.parse_type()?;
self.expect(Token::Semicolon)?; self.expect(Token::Semicolon)?;

186
parser/src/test.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -14,17 +14,18 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::{tokenizer, ParserContext, SpannedToken};
use leo_ast::{Expression, ExpressionStatement, Statement, ValueExpression}; use leo_ast::{Expression, ExpressionStatement, Statement, ValueExpression};
use leo_errors::Span; use leo_errors::{emitter::Handler, LeoError};
use leo_span::{symbol::create_session_if_not_set_then, Span};
use leo_test_framework::{ use leo_test_framework::{
runner::{Namespace, ParseType, Runner}, runner::{Namespace, ParseType, Runner},
Test, Test,
}; };
use serde::Serialize;
use serde_yaml::Value; use serde_yaml::Value;
use tokenizer::Token; use tokenizer::Token;
use crate::{tokenizer, ParserContext};
struct TokenNamespace; struct TokenNamespace;
impl Namespace for TokenNamespace { impl Namespace for TokenNamespace {
@ -33,18 +34,19 @@ impl Namespace for TokenNamespace {
} }
fn run_test(&self, test: Test) -> Result<Value, String> { fn run_test(&self, test: Test) -> Result<Value, String> {
let output = tokenizer::tokenize("test", test.content.into()); create_session_if_not_set_then(|_| {
output tokenizer::tokenize("test", test.content.into())
.map(|tokens| { .map(|tokens| {
Value::String( Value::String(
tokens tokens
.into_iter() .into_iter()
.map(|x| x.to_string()) .map(|x| x.to_string())
.collect::<Vec<String>>() .collect::<Vec<String>>()
.join(","), .join(","),
) )
}) })
.map_err(|x| x.to_string()) .map_err(|x| x.to_string())
})
} }
} }
@ -60,6 +62,37 @@ fn not_fully_consumed(tokens: &mut ParserContext) -> Result<(), String> {
Err(out) Err(out)
} }
fn with_handler<T>(
tokens: Vec<SpannedToken>,
logic: impl FnOnce(&mut ParserContext<'_>) -> Result<T, LeoError>,
) -> Result<T, String> {
let (handler, buf) = Handler::new_with_buf();
let mut tokens = ParserContext::new(&handler, tokens);
let parsed = handler
.extend_if_error(logic(&mut tokens))
.map_err(|_| buf.extract().to_string())?;
not_fully_consumed(&mut tokens)?;
Ok(parsed)
}
fn implcit_value_expr() -> Expression {
Expression::Value(ValueExpression::Implicit("".into(), Span::default()))
}
fn tokenize(test: Test) -> Result<Vec<SpannedToken>, String> {
tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())
}
fn all_are_comments(tokens: &[SpannedToken]) -> bool {
tokens
.iter()
.all(|x| matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_)))
}
fn yaml_or_fail<T: Serialize>(value: T) -> Value {
serde_yaml::to_value(value).expect("serialization failed")
}
struct ParseExpressionNamespace; struct ParseExpressionNamespace;
impl Namespace for ParseExpressionNamespace { impl Namespace for ParseExpressionNamespace {
@ -68,23 +101,13 @@ impl Namespace for ParseExpressionNamespace {
} }
fn run_test(&self, test: Test) -> Result<Value, String> { fn run_test(&self, test: Test) -> Result<Value, String> {
let tokenizer = tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())?; create_session_if_not_set_then(|_| {
if tokenizer let tokenizer = tokenize(test)?;
.iter() if all_are_comments(&tokenizer) {
.all(|x| matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_))) return Ok(yaml_or_fail(implcit_value_expr()));
{ }
return Ok(serde_yaml::to_value(&Expression::Value(ValueExpression::Implicit( with_handler(tokenizer, |p| p.parse_expression()).map(yaml_or_fail)
"".into(), })
Span::default(),
)))
.expect("serialization failed"));
}
let mut tokens = ParserContext::new(tokenizer);
let parsed = tokens.parse_expression().map_err(|x| x.to_string())?;
not_fully_consumed(&mut tokens)?;
Ok(serde_yaml::to_value(&parsed).expect("serialization failed"))
} }
} }
@ -96,23 +119,16 @@ impl Namespace for ParseStatementNamespace {
} }
fn run_test(&self, test: Test) -> Result<Value, String> { fn run_test(&self, test: Test) -> Result<Value, String> {
let tokenizer = tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())?; create_session_if_not_set_then(|_| {
if tokenizer let tokenizer = tokenize(test)?;
.iter() if all_are_comments(&tokenizer) {
.all(|x| matches!(x.token, Token::CommentLine(_) | Token::CommentBlock(_))) return Ok(yaml_or_fail(Statement::Expression(ExpressionStatement {
{ expression: implcit_value_expr(),
return Ok(serde_yaml::to_value(&Statement::Expression(ExpressionStatement { span: Span::default(),
expression: Expression::Value(ValueExpression::Implicit("".into(), Span::default())), })));
span: Span::default(), }
})) with_handler(tokenizer, |p| p.parse_statement()).map(yaml_or_fail)
.expect("serialization failed")); })
}
let mut tokens = ParserContext::new(tokenizer);
let parsed = tokens.parse_statement().map_err(|x| x.to_string())?;
not_fully_consumed(&mut tokens)?;
Ok(serde_yaml::to_value(&parsed).expect("serialization failed"))
} }
} }
@ -124,13 +140,72 @@ impl Namespace for ParseNamespace {
} }
fn run_test(&self, test: Test) -> Result<Value, String> { fn run_test(&self, test: Test) -> Result<Value, String> {
let tokenizer = tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())?; create_session_if_not_set_then(|_| with_handler(tokenize(test)?, |p| p.parse_program()).map(yaml_or_fail))
let mut tokens = ParserContext::new(tokenizer); }
}
let parsed = tokens.parse_program().map_err(|x| x.to_string())?; struct SerializeNamespace;
not_fully_consumed(&mut tokens)?;
Ok(serde_yaml::to_value(&parsed).expect("serialization failed")) // Helper functions to recursively filter keys from AST JSON.
// Redeclaring here since we don't want to make this public.
fn remove_key_from_json(value: &mut serde_json::Value, key: &str) {
match value {
serde_json::value::Value::Object(map) => {
map.remove(key);
for val in map.values_mut() {
remove_key_from_json(val, key);
}
}
serde_json::value::Value::Array(values) => {
for val in values.iter_mut() {
remove_key_from_json(val, key);
}
}
_ => (),
}
}
// Helper function to normalize AST
// Redeclaring here because we don't want to make this public
fn normalize_json_value(value: serde_json::Value) -> serde_json::Value {
match value {
serde_json::Value::Array(vec) => {
let orig_length = vec.len();
let mut new_vec: Vec<serde_json::Value> = vec
.into_iter()
.filter(|v| !matches!(v, serde_json::Value::Object(map) if map.is_empty()))
.map(normalize_json_value)
.collect();
if orig_length == 2 && new_vec.len() == 1 {
new_vec.pop().unwrap()
} else {
serde_json::Value::Array(new_vec)
}
}
serde_json::Value::Object(map) => {
serde_json::Value::Object(map.into_iter().map(|(k, v)| (k, normalize_json_value(v))).collect())
}
_ => value,
}
}
impl Namespace for SerializeNamespace {
fn parse_type(&self) -> ParseType {
ParseType::Whole
}
fn run_test(&self, test: Test) -> Result<Value, String> {
create_session_if_not_set_then(|_| {
let tokenizer = tokenize(test)?;
let parsed = with_handler(tokenizer, |p| p.parse_program())?;
let mut json = serde_json::to_value(parsed).expect("failed to convert to json value");
remove_key_from_json(&mut json, "span");
json = normalize_json_value(json);
Ok(serde_json::from_value::<serde_yaml::Value>(json).expect("failed serialization"))
})
} }
} }
@ -143,6 +218,7 @@ impl Runner for TestRunner {
"ParseStatement" => Box::new(ParseStatementNamespace), "ParseStatement" => Box::new(ParseStatementNamespace),
"ParseExpression" => Box::new(ParseExpressionNamespace), "ParseExpression" => Box::new(ParseExpressionNamespace),
"Token" => Box::new(TokenNamespace), "Token" => Box::new(TokenNamespace),
"Serialize" => Box::new(SerializeNamespace),
_ => return None, _ => return None,
}) })
} }

12
parser/src/tokenizer/lexer.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -15,7 +15,8 @@
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::tokenizer::{Char, Token}; use crate::tokenizer::{Char, Token};
use leo_errors::Span; use leo_span::{Span, Symbol};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tendril::StrTendril; use tendril::StrTendril;
@ -352,7 +353,7 @@ impl Token {
// else if let Some(len) = eat(input, "&=") { // else if let Some(len) = eat(input, "&=") {
// return (len, Some(Token::BitAndEq)); // return (len, Some(Token::BitAndEq));
// } // }
// return (1, Some(Token::BitAnd)); return (1, Some(Token::Ampersand));
} }
b'(' => return (1, Some(Token::LeftParen)), b'(' => return (1, Some(Token::LeftParen)),
b')' => return (1, Some(Token::RightParen)), b')' => return (1, Some(Token::RightParen)),
@ -388,9 +389,6 @@ impl Token {
return (len, Some(Token::DotDotDot)); return (len, Some(Token::DotDotDot));
} else if let Some(len) = eat(input, "..") { } else if let Some(len) = eat(input, "..") {
return (len, Some(Token::DotDot)); return (len, Some(Token::DotDot));
} else if let Some(len) = eat(input, ".len()") {
// FIXME: remove this code once we allow method calls
return (len, Some(Token::LengthOf));
} }
return (1, Some(Token::Dot)); return (1, Some(Token::Dot));
} }
@ -526,7 +524,7 @@ impl Token {
"u32" => Token::U32, "u32" => Token::U32,
"u64" => Token::U64, "u64" => Token::U64,
"u128" => Token::U128, "u128" => Token::U128,
_ => Token::Ident(ident), _ => Token::Ident(Symbol::intern(&ident)),
}), }),
); );
} }

104
parser/src/tokenizer/mod.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -28,7 +28,8 @@ pub(crate) use self::token::*;
pub(crate) mod lexer; pub(crate) mod lexer;
pub(crate) use self::lexer::*; pub(crate) use self::lexer::*;
use leo_errors::{LeoError, ParserError, Span}; use leo_errors::{LeoError, ParserError};
use leo_span::Span;
use tendril::StrTendril; use tendril::StrTendril;
@ -112,30 +113,31 @@ pub(crate) fn tokenize(path: &str, input: StrTendril) -> Result<Vec<SpannedToken
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use leo_span::symbol::create_session_if_not_set_then;
#[test] #[test]
fn test_tokenizer() { fn test_tokenizer() {
// & create_session_if_not_set_then(|_| {
// &= // &=
// | // |
// |= // |=
// ^ // ^
// ^= // ^=
// ~ // ~
// << // <<
// <<= // <<=
// >> // >>
// >>= // >>=
// >>> // >>>
// >>>= // >>>=
// % // %
// %= // %=
// ||= // ||=
// &&= // &&=
let tokens = tokenize( let tokens = tokenize(
"test_path", "test_path",
r#" r#"
"test" "test"
"test{}test" "test{}test"
"test{}" "test{}"
@ -170,6 +172,7 @@ mod tests {
input input
let let
mut mut
&
return return
static static
string string
@ -223,25 +226,27 @@ mod tests {
// test // test
/* test */ /* test */
//"# //"#
.into(), .into(),
) )
.unwrap(); .unwrap();
let mut output = String::new(); let mut output = String::new();
for SpannedToken { token, .. } in tokens.iter() { for SpannedToken { token, .. } in tokens.iter() {
output += &format!("{} ", token.to_string()); output += &format!("{} ", token.to_string());
} }
// & &= | |= ^ ^= ~ << <<= >> >>= >>> >>>= % %= ||= &&= // & &= | |= ^ ^= ~ << <<= >> >>= >>> >>>= % %= ||= &&=
assert_eq!( assert_eq!(
output, output,
r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut & return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
/* test */ // "# /* test */ // "#
); );
});
} }
#[test] #[test]
fn test_spans() { fn test_spans() {
let raw = r#" create_session_if_not_set_then(|_| {
let raw = r#"
test test
// test // test
test test
@ -251,20 +256,21 @@ mod tests {
test */ test */
test test
"#; "#;
let tokens = tokenize("test_path", raw.into()).unwrap(); let tokens = tokenize("test_path", raw.into()).unwrap();
let mut line_indicies = vec![0]; let mut line_indicies = vec![0];
for (i, c) in raw.chars().enumerate() { for (i, c) in raw.chars().enumerate() {
if c == '\n' { if c == '\n' {
line_indicies.push(i + 1); line_indicies.push(i + 1);
}
} }
} for token in tokens.iter() {
for token in tokens.iter() { let token_raw = token.token.to_string();
let token_raw = token.token.to_string(); let start = line_indicies.get(token.span.line_start - 1).unwrap();
let start = line_indicies.get(token.span.line_start - 1).unwrap(); let stop = line_indicies.get(token.span.line_stop - 1).unwrap();
let stop = line_indicies.get(token.span.line_stop - 1).unwrap(); let original = &raw[*start + token.span.col_start - 1..*stop + token.span.col_stop - 1];
let original = &raw[*start + token.span.col_start - 1..*stop + token.span.col_stop - 1]; assert_eq!(original, &token_raw);
assert_eq!(original, &token_raw); }
} // println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
// println!("{}", serde_json::to_string_pretty(&tokens).unwrap()); })
} }
} }

75
parser/src/tokenizer/token.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -14,6 +14,8 @@
// You should have received a copy of the GNU General Public License // You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>. // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use leo_span::{sym, Symbol};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::fmt; use std::fmt;
use tendril::StrTendril; use tendril::StrTendril;
@ -48,14 +50,14 @@ impl fmt::Display for Char {
pub enum Token { pub enum Token {
// Lexical Grammar // Lexical Grammar
// Literals // Literals
CommentLine(#[serde(with = "leo_errors::common::tendril_json")] StrTendril), CommentLine(#[serde(with = "leo_span::tendril_json")] StrTendril),
CommentBlock(#[serde(with = "leo_errors::common::tendril_json")] StrTendril), CommentBlock(#[serde(with = "leo_span::tendril_json")] StrTendril),
StringLit(Vec<leo_ast::Char>), StringLit(Vec<leo_ast::Char>),
Ident(#[serde(with = "leo_errors::common::tendril_json")] StrTendril), Ident(Symbol),
Int(#[serde(with = "leo_errors::common::tendril_json")] StrTendril), Int(#[serde(with = "leo_span::tendril_json")] StrTendril),
True, True,
False, False,
AddressLit(#[serde(with = "leo_errors::common::tendril_json")] StrTendril), AddressLit(#[serde(with = "leo_span::tendril_json")] StrTendril),
CharLit(Char), CharLit(Char),
// Symbols // Symbols
@ -127,6 +129,7 @@ pub enum Token {
As, As,
Circuit, Circuit,
Console, Console,
/// Const variable and a const function.
Const, Const,
Else, Else,
For, For,
@ -135,16 +138,14 @@ pub enum Token {
In, In,
Let, Let,
Mut, Mut,
/// Represents `&`.
/// Used for `Reference` and `BitAnd`.
Ampersand,
Return, Return,
Static, Static,
Type, Type,
// Not yet in ABNF // Not yet in ABNF
// arr.len() token - hacky zone
LengthOf,
// Not yet in ABNF
// BitAnd,
// BitAndEq, // BitAndEq,
// BitOr, // BitOr,
// BitOrEq, // BitOrEq,
@ -192,13 +193,13 @@ pub const KEYWORD_TOKENS: &[Token] = &[
Token::Input, Token::Input,
Token::Let, Token::Let,
Token::Mut, Token::Mut,
Token::Ampersand,
Token::Return, Token::Return,
Token::BigSelf, Token::BigSelf,
Token::LittleSelf, Token::LittleSelf,
Token::Static, Token::Static,
Token::True, Token::True,
Token::Type, Token::Type,
Token::LengthOf,
Token::U8, Token::U8,
Token::U16, Token::U16,
Token::U32, Token::U32,
@ -207,11 +208,52 @@ pub const KEYWORD_TOKENS: &[Token] = &[
]; ];
impl Token { impl Token {
///
/// Returns `true` if the `self` token equals a Leo keyword. /// Returns `true` if the `self` token equals a Leo keyword.
///
pub fn is_keyword(&self) -> bool { pub fn is_keyword(&self) -> bool {
KEYWORD_TOKENS.iter().any(|x| x == self) KEYWORD_TOKENS.contains(self)
}
/// Converts `self` to the corresponding `Symbol` if it `is_keyword`.
pub fn keyword_to_symbol(&self) -> Option<Symbol> {
Some(match self {
Token::Address => sym::address,
Token::As => sym::As,
Token::Bool => sym::bool,
Token::Char => sym::char,
Token::Circuit => sym::circuit,
Token::Console => sym::console,
Token::Const => sym::Const,
Token::Else => sym::Else,
Token::False => sym::False,
Token::Field => sym::field,
Token::For => sym::For,
Token::Function => sym::function,
Token::Group => sym::group,
Token::I8 => sym::i8,
Token::I16 => sym::i16,
Token::I32 => sym::i32,
Token::I64 => sym::i64,
Token::I128 => sym::i128,
Token::If => sym::If,
Token::Import => sym::import,
Token::In => sym::In,
Token::Input => sym::input,
Token::Let => sym::Let,
Token::Mut => sym::Mut,
Token::Ampersand => sym::Ampersand,
Token::Return => sym::Return,
Token::BigSelf => sym::SelfUpper,
Token::LittleSelf => sym::SelfLower,
Token::Static => sym::Static,
Token::True => sym::True,
Token::Type => sym::Type,
Token::U8 => sym::u8,
Token::U16 => sym::u16,
Token::U32 => sym::u32,
Token::U64 => sym::u64,
Token::U128 => sym::u128,
_ => return None,
})
} }
} }
@ -307,12 +349,11 @@ impl fmt::Display for Token {
In => write!(f, "in"), In => write!(f, "in"),
Let => write!(f, "let"), Let => write!(f, "let"),
Mut => write!(f, "mut"), Mut => write!(f, "mut"),
Ampersand => write!(f, "&"), // Used for `Reference` and `BitAnd`
Return => write!(f, "return"), Return => write!(f, "return"),
Static => write!(f, "static"), Static => write!(f, "static"),
Type => write!(f, "type"), Type => write!(f, "type"),
LengthOf => write!(f, ".len()"), // FIXME
Eof => write!(f, ""), Eof => write!(f, ""),
// BitAnd => write!(f, "&"),
// BitAndEq => write!(f, "&="), // BitAndEq => write!(f, "&="),
// BitOr => write!(f, "|"), // BitOr => write!(f, "|"),
// BitOrEq => write!(f, "|="), // BitOrEq => write!(f, "|="),

17
parser/tests/mod.rs
View File

@ -1,17 +0,0 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
mod serialization;

1162
parser/tests/serialization/expected_leo_ast/linear_regression.json
View File

File diff suppressed because it is too large Load Diff

100
parser/tests/serialization/expected_leo_ast/one_plus_one.json
View File

@ -1,100 +0,0 @@
{
"name": "",
"expected_input": [],
"import_statements": [],
"imports": {},
"aliases": {},
"circuits": {},
"global_consts": {},
"functions": {
"{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main() -> u8 {\\\"}\"}": {
"annotations": [],
"identifier": "{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main() -> u8 {\\\"}\"}",
"input": [],
"output": {
"IntegerType": "U8"
},
"block": {
"statements": [
{
"Return": {
"expression": {
"Binary": {
"left": {
"Value": {
"Integer": [
"U8",
"1",
{
"span" : {
"line_start": 2,
"line_stop": 2,
"col_start": 12,
"col_stop": 15,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
]
}
},
"right": {
"Value": {
"Integer": [
"U8",
"1",
{
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 18,
"col_stop": 21,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
]
}
},
"op": "Add",
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 12,
"col_stop": 21,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 5,
"col_stop": 21,
"path": "",
"content": " return 1u8 + 1u8;"
}
}
}
],
"span": {
"line_start": 1,
"line_stop": 3,
"col_start": 23,
"col_stop": 2,
"path": "",
"content": "function main() -> u8 {\n ...\n}"
}
},
"span": {
"line_start": 1,
"line_stop": 3,
"col_start": 1,
"col_stop": 2,
"path": "",
"content": "function main() -> u8 {\n ...\n}"
}
}
}
}

1189
parser/tests/serialization/expected_leo_ast/palindrome.json
View File

File diff suppressed because it is too large Load Diff

299
parser/tests/serialization/expected_leo_ast/pedersen_hash.json
View File

@ -1,299 +0,0 @@
{
"name": "",
"expected_input": [],
"import_statements": [],
"imports": {},
"aliases": {},
"circuits": {
"{\"name\":\"PedersenHash\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":9,\\\"col_stop\\\":21,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"circuit PedersenHash {\\\"}\"}": {
"circuit_name": "{\"name\":\"PedersenHash\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":9,\\\"col_stop\\\":21,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"circuit PedersenHash {\\\"}\"}",
"core_mapping": null,
"members": [
{
"CircuitVariable": [
"{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":2,\\\"line_stop\\\":2,\\\"col_start\\\":5,\\\"col_stop\\\":15,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" parameters: [group; 256];\\\"}\"}",
{
"Array": [
"Group",
[
{
"value": "256"
}
]
]
}
]
},
{
"CircuitFunction": {
"annotations": [],
"identifier": "{\"name\":\"new\",\"span\":\"{\\\"line_start\\\":5,\\\"line_stop\\\":5,\\\"col_start\\\":14,\\\"col_stop\\\":17,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function new(parameters: [group; 256]) -> Self {\\\"}\"}",
"input": [
{
"Variable": {
"identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":5,\\\"line_stop\\\":5,\\\"col_start\\\":18,\\\"col_stop\\\":28,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function new(parameters: [group; 256]) -> Self {\\\"}\"}",
"const_": false,
"mutable": true,
"type_": {
"Array": [
"Group",
[
{
"value": "256"
}
]
]
}
}
}
],
"output": "SelfType",
"block": {
"statements": [
{
"Return": {
"expression": {
"CircuitInit": {
"name": "{\"name\":\"Self\",\"span\":\"{\\\"line_start\\\":6,\\\"line_stop\\\":6,\\\"col_start\\\":16,\\\"col_stop\\\":20,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return Self { parameters: parameters };\\\"}\"}",
"members": [
{
"identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":6,\\\"line_stop\\\":6,\\\"col_start\\\":23,\\\"col_stop\\\":33,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return Self { parameters: parameters };\\\"}\"}",
"expression": {
"Identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":6,\\\"line_stop\\\":6,\\\"col_start\\\":35,\\\"col_stop\\\":45,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return Self { parameters: parameters };\\\"}\"}"
}
}
]
}
}
}
}
]
}
}
},
{
"CircuitFunction": {
"annotations": [],
"identifier": "{\"name\":\"hash\",\"span\":\"{\\\"line_start\\\":9,\\\"line_stop\\\":9,\\\"col_start\\\":14,\\\"col_stop\\\":18,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function hash(self, bits: [bool; 256]) -> group {\\\"}\"}",
"input": [
{
"SelfKeyword": "{\"name\":\"self\",\"span\":\"{\\\"line_start\\\":9,\\\"line_stop\\\":9,\\\"col_start\\\":19,\\\"col_stop\\\":23,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function hash(self, bits: [bool; 256]) -> group {\\\"}\"}"
},
{
"Variable": {
"identifier": "{\"name\":\"bits\",\"span\":\"{\\\"line_start\\\":9,\\\"line_stop\\\":9,\\\"col_start\\\":25,\\\"col_stop\\\":29,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" function hash(self, bits: [bool; 256]) -> group {\\\"}\"}",
"const_": false,
"mutable": true,
"type_": {
"Array": [
"Boolean",
[
{
"value": "256"
}
]
]
}
}
}
],
"output": "Group",
"block": {
"statements": [
{
"Definition": {
"declaration_type": "Let",
"variable_names": [
{
"mutable": true,
"identifier": "{\"name\":\"digest\",\"span\":\"{\\\"line_start\\\":10,\\\"line_stop\\\":10,\\\"col_start\\\":13,\\\"col_stop\\\":19,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" let digest: group = 0group;\\\"}\"}"
}
],
"type_": "Group",
"value": {
"Value": {
"Group": {
"Single": "0"
}
}
}
}
},
{
"Iteration": {
"variable": "{\"name\":\"i\",\"span\":\"{\\\"line_start\\\":11,\\\"line_stop\\\":11,\\\"col_start\\\":13,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" for i in 0..256 {\\\"}\"}",
"start": {
"Value": {
"Implicit": "0"
}
},
"stop": {
"Value": {
"Implicit": "256"
}
},
"inclusive": false,
"block": {
"statements": [
{
"Conditional": {
"condition": {
"ArrayAccess": {
"array": {
"Identifier": "{\"name\":\"bits\",\"span\":\"{\\\"line_start\\\":12,\\\"line_stop\\\":12,\\\"col_start\\\":16,\\\"col_stop\\\":20,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" if bits[i] {\\\"}\"}"
},
"index": {
"Identifier": "{\"name\":\"i\",\"span\":\"{\\\"line_start\\\":12,\\\"line_stop\\\":12,\\\"col_start\\\":21,\\\"col_stop\\\":22,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" if bits[i] {\\\"}\"}"
}
}
},
"block": {
"statements": [
{
"Assign": {
"operation": "Add",
"assignee": {
"identifier": "{\"name\":\"digest\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":17,\\\"col_stop\\\":23,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}",
"accesses": []
},
"value": {
"ArrayAccess": {
"array": {
"CircuitMemberAccess": {
"circuit": {
"Identifier": "{\"name\":\"self\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":27,\\\"col_stop\\\":31,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}"
},
"name": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":32,\\\"col_stop\\\":42,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}",
"type_": null
}
},
"index": {
"Identifier": "{\"name\":\"i\",\"span\":\"{\\\"line_start\\\":13,\\\"line_stop\\\":13,\\\"col_start\\\":43,\\\"col_stop\\\":44,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" digest += self.parameters[i];\\\"}\"}"
}
}
}
}
}
]
},
"next": null
}
}
]
}
}
},
{
"Return": {
"expression": {
"Identifier": "{\"name\":\"digest\",\"span\":\"{\\\"line_start\\\":16,\\\"line_stop\\\":16,\\\"col_start\\\":16,\\\"col_stop\\\":22,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return digest;\\\"}\"}"
}
}
}
]
}
}
}
]
}
},
"global_consts": {},
"functions": {
"{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}": {
"annotations": [],
"identifier": "{\"name\":\"main\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":10,\\\"col_stop\\\":14,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}",
"input": [
{
"Variable": {
"identifier": "{\"name\":\"hash_input\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":15,\\\"col_stop\\\":25,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}",
"const_": false,
"mutable": true,
"type_": {
"Array": [
"Boolean",
[
{
"value": "256"
}
]
]
}
}
},
{
"Variable": {
"identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":21,\\\"line_stop\\\":21,\\\"col_start\\\":46,\\\"col_stop\\\":56,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\"function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {\\\"}\"}",
"const_": true,
"mutable": false,
"type_": {
"Array": [
"Group",
[
{
"value": "256"
}
]
]
}
}
}
],
"output": "Group",
"block": {
"statements": [
{
"Definition": {
"declaration_type": "Const",
"variable_names": [
{
"mutable": false,
"identifier": "{\"name\":\"pedersen\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":11,\\\"col_stop\\\":19,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
}
],
"type_": null,
"value": {
"Call": {
"function": {
"CircuitStaticFunctionAccess": {
"circuit": {
"Identifier": "{\"name\":\"PedersenHash\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":22,\\\"col_stop\\\":34,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
},
"name": "{\"name\":\"new\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":36,\\\"col_stop\\\":39,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
}
},
"arguments": [
{
"Identifier": "{\"name\":\"parameters\",\"span\":\"{\\\"line_start\\\":22,\\\"line_stop\\\":22,\\\"col_start\\\":40,\\\"col_stop\\\":50,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" const pedersen = PedersenHash::new(parameters);\\\"}\"}"
}
]
}
}
}
},
{
"Return": {
"expression": {
"Call": {
"function": {
"CircuitMemberAccess": {
"circuit": {
"Identifier": "{\"name\":\"pedersen\",\"span\":\"{\\\"line_start\\\":23,\\\"line_stop\\\":23,\\\"col_start\\\":12,\\\"col_stop\\\":20,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return pedersen.hash(hash_input);\\\"}\"}"
},
"name": "{\"name\":\"hash\",\"span\":\"{\\\"line_start\\\":23,\\\"line_stop\\\":23,\\\"col_start\\\":21,\\\"col_stop\\\":25,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return pedersen.hash(hash_input);\\\"}\"}",
"type_": null
}
},
"arguments": [
{
"Identifier": "{\"name\":\"hash_input\",\"span\":\"{\\\"line_start\\\":23,\\\"line_stop\\\":23,\\\"col_start\\\":26,\\\"col_stop\\\":36,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" return pedersen.hash(hash_input);\\\"}\"}"
}
]
}
}
}
}
]
}
}
}
}

1209
parser/tests/serialization/expected_leo_ast/silly_sudoku.json
View File

File diff suppressed because it is too large Load Diff

6
parser/tests/serialization/json.rs
View File

@ -1,4 +1,4 @@
// Copyright (C) 2019-2021 Aleo Systems Inc. // Copyright (C) 2019-2022 Aleo Systems Inc.
// This file is part of the Leo library. // This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify // The Leo library is free software: you can redistribute it and/or modify
@ -17,7 +17,7 @@
use leo_ast::Ast; use leo_ast::Ast;
#[cfg(not(feature = "ci_skip"))] #[cfg(not(feature = "ci_skip"))]
use leo_ast::Program; use leo_ast::Program;
use leo_errors::{LeoError, Result}; use leo_errors::{emitter::Handler, LeoError, Result};
use std::fs::File; use std::fs::File;
use std::io::BufReader; use std::io::BufReader;
@ -28,7 +28,7 @@ fn to_ast(program_filepath: &Path) -> Result<Ast> {
let program_string = std::fs::read_to_string(program_filepath).expect("failed to open test"); let program_string = std::fs::read_to_string(program_filepath).expect("failed to open test");
// Parses the Leo file and constructs a leo ast. // Parses the Leo file and constructs a leo ast.
leo_parser::parse_ast("", &program_string) leo_parser::parse_ast(&Handler::default(), "", &program_string)
} }
fn setup() { fn setup() {

7
parser/tests/serialization/leo/deprecated_error.leo
View File

@ -1,7 +0,0 @@
function main() {
return 1 + 1;
}
test function old {
console.log("old");
}

65
parser/tests/serialization/leo/linear_regression.leo
View File

@ -1,65 +0,0 @@
circuit Point {
x: i32,
y: i32,
function new(x: i32, y: i32) -> Self {
return Self { x, y };
}
}
circuit LinearRegression {
points: [Point; 5],
// Instantiates a linear regression circuit.
function new(points: [Point; 5]) -> Self {
return Self { points };
}
// Return the slope of the linear regression.
function slope(self) -> i32 {
let num_points = 5i32;
// Calculate the sums.
let x_sum = 0i32;
let y_sum = 0i32;
let xy_sum = 0i32;
let x2_sum = 0i32;
for i in 0..5 {
x_sum += self.points[i].x;
y_sum += self.points[i].y;
xy_sum += self.points[i].x * self.points[i].y;
x2_sum += self.points[i].x * self.points[i].x;
}
let numerator = (num_points * xy_sum) - (x_sum * y_sum);
let denominator = (num_points * x2_sum) - (x_sum * x_sum);
let slope = numerator / denominator;
return slope;
}
// Return the offset of the linear regression.
function offset(self, slope: i32) -> i32 {
let num_points = 5i32;
// Calculate the sum.
let x_sum = 0i32;
let y_sum = 0i32;
for i in 0..5 {
x_sum += self.points[i].x;
y_sum += self.points[i].y;
}
return (y_sum - slope * x_sum) / num_points;
}
}
function main (x: i32, y: i32) -> [i32; 2] {
let points: [Point; 5] = [
Point{x: x + 1, y: y + 1},
Point{x: x + 2, y: y + 2},
Point{x: x + 3, y: y + 3},
Point{x: x + 4, y: y + 4},
Point{x: x + 5, y: y + 5}
];
let reg = LinearRegression::new(points);
let slope = reg.slope();
let offset = reg.offset(slope);
return [slope, offset];
}

3
parser/tests/serialization/leo/one_plus_one.leo
View File

@ -1,3 +0,0 @@
function main() -> u8 {
return 1u8 + 1u8;
}

59
parser/tests/serialization/leo/palindrome.leo
View File

@ -1,59 +0,0 @@
// This Program takes in any 20-byte low register string and tells
// whether a string is a palindrome ignoring any spaces.
function main(str: [char; 20]) -> bool {
return is_palindrome(str);
}
function is_palindrome(str: [char; 20]) -> bool {
const str_len = 20u32; // saving const for convenience
// By default we assume that input is a palindrome.
let result = true;
let processed = 0u8;
for start in 0..(str_len / 2) {
let start_sym = str[start];
if start_sym != ' ' {
let skipped = 0u8;
let end_empty = 0u8;
let end_sym = ' ';
for end in (str_len - 1)..start {
if str[end] != ' ' && skipped == processed && end_sym == ' ' {
end_sym = str[end];
} else {
end_empty = end_empty + 1;
if str[end] != ' ' {
skipped = skipped + 1;
}
}
}
// If there are symbols left to the right from the start.
if end_sym != ' ' {
console.log("Comparing: {} ? {}", start_sym, end_sym);
if result {
result = (start_sym == end_sym);
}
processed = processed + 1;
}
}
}
console.log("Result is: {}", result);
return result;
}
@test
function test_is_palindrome() {
console.assert(is_palindrome("a b a "));
console.assert(is_palindrome("😀😀😀😀😀 😀😀😀😀😀"));
console.assert(is_palindrome("borrow or rob "));
console.assert(is_palindrome("bbbb aaaa aaaa bbbb"));
console.assert(is_palindrome("aaaaaaaaaaaaaaaaaaaa"));
console.assert(is_palindrome("taco cat "));
}

1
parser/tests/serialization/leo/parser_error.leo
View File

@ -1 +0,0 @@
invalid

25
parser/tests/serialization/leo/pedersen_hash.leo
View File

@ -1,25 +0,0 @@
circuit PedersenHash {
parameters: [group; 256];
// Instantiates a Pedersen hash circuit
function new(parameters: [group; 256]) -> Self {
return Self { parameters: parameters };
}
function hash(self, bits: [bool; 256]) -> group {
let digest: group = 0group;
for i in 0..256 {
if bits[i] {
digest += self.parameters[i];
}
}
return digest;
}
}
// The 'pedersen-hash' main function.
function main(hash_input: [bool; 256], const parameters: [group; 256]) -> group {
const pedersen = PedersenHash::new(parameters);
return pedersen.hash(hash_input);
}

71
parser/tests/serialization/leo/silly_sudoku.leo
View File

@ -1,71 +0,0 @@
import lib.SillySudoku;
// The `silly-sudoku` main function
function main(puzzle: [u8; (3, 3)], answer: [u8; (3, 3)]) -> bool {
console.log("Starting Sudoku solver...");
console.log("{}", puzzle);
// Instantiate the Sudoku puzzle.
let sudoku = SillySudoku { puzzle_grid: puzzle };
console.log("Checking Sudoku answer...");
console.log("{}", answer);
// Evaluate the Sudoku puzzle with the given answer.
let result = sudoku.solve(answer);
console.log("The answer is {}.", result);
return result;
}
// Tests that the `silly-sudoku` circuit outputs true on a correct answer.
@test
function test_solve_pass() {
let puzzle: [u8; (3, 3)] = [[0, 2, 0],
[0, 0, 6],
[0, 8, 9]];
let answer: [u8; (3, 3)] = [[1, 2, 3],
[4, 5, 6],
[7, 8, 9]];
// Runs the Sudoku checker.
let result = main(puzzle, answer);
// Expects the result to be true.
console.assert(true == result);
}
// Tests that the `silly-sudoku` circuit outputs false on an incorrect answer.
@test
function test_solve_fail() {
let puzzle: [u8; (3, 3)] = [[0, 2, 0],
[0, 0, 6],
[0, 8, 0]];
let answer: [u8; (3, 3)] = [[1, 2, 3],
[4, 5, 6],
[7, 8, 8]]; // We have an extra `8` in this column!
// Runs the Sudoku checker.
let result = main(puzzle, answer);
// Expects the result to be false.
console.assert(false == result);
}
// Test that the `silly-sudoku` circuit outputs the expected value on a custom test input.
@test(test_input)
function test_solve_with_input(
puzzle: [u8; (3, 3)],
answer: [u8; (3, 3)],
expected: bool
) {
// Runs the Sudoku checker.
let result = main(puzzle, answer);
console.log("expected {}, got {}", expected, result);
console.assert(expected == result);
}

17
parser/tests/serialization/mod.rs
View File

@ -1,17 +0,0 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
mod json;