From 3d1cc9a7359d011353a86a7d10a67c86d08c703b Mon Sep 17 00:00:00 2001
From: gluax <16431709+gluax@users.noreply.github.com>
Date: Tue, 15 Mar 2022 14:53:42 -0700
Subject: [PATCH 01/13] most things moved to a char parser
---
compiler/ast/src/common/positive_number.rs | 6 +-
compiler/ast/src/expression/value.rs | 28 +-
compiler/ast/src/groups/group_coordinate.rs | 6 +-
compiler/ast/src/groups/group_value.rs | 6 +-
compiler/ast/src/input/input_value.rs | 10 +-
compiler/parser/src/parser/context.rs | 3 +-
compiler/parser/src/parser/expression.rs | 13 +-
compiler/parser/src/parser/mod.rs | 4 +-
compiler/parser/src/test.rs | 4 +-
compiler/parser/src/tokenizer/lexer.rs | 412 ++++++--------------
compiler/parser/src/tokenizer/mod.rs | 16 +-
compiler/parser/src/tokenizer/token.rs | 9 +-
leo/errors/src/parser/parser_errors.rs | 9 +
leo/span/src/span.rs | 8 +-
14 files changed, 169 insertions(+), 365 deletions(-)
diff --git a/compiler/ast/src/common/positive_number.rs b/compiler/ast/src/common/positive_number.rs
index a083e45b26..b05416560b 100644
--- a/compiler/ast/src/common/positive_number.rs
+++ b/compiler/ast/src/common/positive_number.rs
@@ -16,21 +16,19 @@
use serde::{Deserialize, Serialize};
use std::fmt;
-use tendril::StrTendril;
/// A number string guaranteed to be positive by the pest grammar.
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq, Hash)]
pub struct PositiveNumber {
/// The string representation of the positive number.
// FIXME(Centril): This should become an `u128`.
- #[serde(with = "leo_span::tendril_json")]
- pub value: StrTendril,
+ pub value: String,
}
impl PositiveNumber {
/// Returns `true` if this number is zero.
pub fn is_zero(&self) -> bool {
- self.value.as_ref().eq("0")
+ self.value.eq("0")
}
}
diff --git a/compiler/ast/src/expression/value.rs b/compiler/ast/src/expression/value.rs
index ccda457d4a..566abd4321 100644
--- a/compiler/ast/src/expression/value.rs
+++ b/compiler/ast/src/expression/value.rs
@@ -14,8 +14,6 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see .
-use tendril::StrTendril;
-
use super::*;
use crate::{Char, CharValue};
@@ -24,37 +22,21 @@ use crate::{Char, CharValue};
pub enum ValueExpression {
// todo: deserialize values here
/// An address literal, e.g., `aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8`.
- Address(
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Address(String, #[serde(with = "leo_span::span_json")] Span),
/// A boolean literal, either `true` or `false`.
- Boolean(
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Boolean(String, #[serde(with = "leo_span::span_json")] Span),
/// A char literal, e.g., `'a'`, representing a single unicode code point.
Char(CharValue),
/// A field literal, e.g., `42field`.
/// That is, a signed number followed by the keyword `field`.
- Field(
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Field(String, #[serde(with = "leo_span::span_json")] Span),
/// A group literal, either product or affine.
/// For example, `42group` or `(12, 52)group`.
Group(Box),
/// A negated non-integer literal, e.g., `-4.2`.
- Implicit(
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Implicit(String, #[serde(with = "leo_span::span_json")] Span),
/// An integer literal, e.g., `42`.
- Integer(
- IntegerType,
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Integer(IntegerType, String, #[serde(with = "leo_span::span_json")] Span),
/// A string literal, e.g., `"foobar"`.
String(Vec, #[serde(with = "leo_span::span_json")] Span),
}
diff --git a/compiler/ast/src/groups/group_coordinate.rs b/compiler/ast/src/groups/group_coordinate.rs
index 8fb39aee29..54960bb2c0 100644
--- a/compiler/ast/src/groups/group_coordinate.rs
+++ b/compiler/ast/src/groups/group_coordinate.rs
@@ -18,16 +18,12 @@ use leo_span::Span;
use serde::{Deserialize, Serialize};
use std::fmt;
-use tendril::StrTendril;
/// A coordinate in a affine group literal.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum GroupCoordinate {
/// A number, e.g., `42`.
- Number(
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Number(String, #[serde(with = "leo_span::span_json")] Span),
/// A sign high recovery, i.e. `+`.
SignHigh,
/// A sign low recovery, i.e., `-`.
diff --git a/compiler/ast/src/groups/group_value.rs b/compiler/ast/src/groups/group_value.rs
index 567527fd90..78e355e49e 100644
--- a/compiler/ast/src/groups/group_value.rs
+++ b/compiler/ast/src/groups/group_value.rs
@@ -19,16 +19,12 @@ use leo_span::Span;
use serde::{Deserialize, Serialize};
use std::fmt;
-use tendril::StrTendril;
/// A group literal.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum GroupValue {
/// Product group literal, e.g., `42group`.
- Single(
- #[serde(with = "leo_span::tendril_json")] StrTendril,
- #[serde(with = "leo_span::span_json")] Span,
- ),
+ Single(String, #[serde(with = "leo_span::span_json")] Span),
/// An affine group literal with (x, y) coordinates.
Tuple(GroupTuple),
}
diff --git a/compiler/ast/src/input/input_value.rs b/compiler/ast/src/input/input_value.rs
index 9ca0a286d0..782b6dec25 100644
--- a/compiler/ast/src/input/input_value.rs
+++ b/compiler/ast/src/input/input_value.rs
@@ -38,22 +38,20 @@ impl TryFrom<(Type, Expression)> for InputValue {
Ok(match value {
(type_, Expression::Value(value)) => {
match (type_, value) {
- (Type::Address, ValueExpression::Address(value, _)) => Self::Address(value.to_string()),
+ (Type::Address, ValueExpression::Address(value, _)) => Self::Address(value),
(Type::Boolean, ValueExpression::Boolean(value, span)) => {
let bool_value = value.parse::().map_err(|_| ParserError::unexpected_eof(&span))?; // TODO: change error
Self::Boolean(bool_value)
}
(Type::Char, ValueExpression::Char(value)) => Self::Char(value),
(Type::Field, ValueExpression::Field(value, _) | ValueExpression::Implicit(value, _)) => {
- Self::Field(value.to_string())
+ Self::Field(value)
}
(Type::Group, ValueExpression::Group(value)) => Self::Group(*value),
- (Type::IntegerType(type_), ValueExpression::Implicit(value, _)) => {
- Self::Integer(type_, value.to_string())
- }
+ (Type::IntegerType(type_), ValueExpression::Implicit(value, _)) => Self::Integer(type_, value),
(Type::IntegerType(expected), ValueExpression::Integer(actual, value, span)) => {
if expected == actual {
- Self::Integer(expected, value.to_string())
+ Self::Integer(expected, value)
} else {
return Err(InputError::unexpected_type(expected.to_string(), actual, &span).into());
}
diff --git a/compiler/parser/src/parser/context.rs b/compiler/parser/src/parser/context.rs
index 4c7479311d..44431a8d0a 100644
--- a/compiler/parser/src/parser/context.rs
+++ b/compiler/parser/src/parser/context.rs
@@ -22,7 +22,6 @@ use leo_errors::{LeoError, ParserError, Result};
use leo_span::{Span, Symbol};
use std::{borrow::Cow, unreachable};
-use tendril::format_tendril;
/// Stores a program in tokenized format plus additional context.
/// May be converted into a [`Program`] AST by parsing all tokens.
@@ -182,7 +181,7 @@ impl<'a> ParserContext<'a> {
return None;
}
*i -= 1;
- GroupCoordinate::Number(format_tendril!("-{}", value), span.clone())
+ GroupCoordinate::Number(format!("-{}", value), span.clone())
}
_ => GroupCoordinate::SignLow,
},
diff --git a/compiler/parser/src/parser/expression.rs b/compiler/parser/src/parser/expression.rs
index 7fdb919966..58d0b4a4c2 100644
--- a/compiler/parser/src/parser/expression.rs
+++ b/compiler/parser/src/parser/expression.rs
@@ -19,8 +19,6 @@ use super::*;
use leo_errors::{ParserError, Result};
use leo_span::sym;
-use tendril::format_tendril;
-
const INT_TYPES: &[Token] = &[
Token::I8,
Token::I16,
@@ -253,17 +251,10 @@ impl ParserContext<'_> {
// hack for const signed integer overflow issues
if matches!(operation, UnaryOperation::Negate) {
if let Expression::Value(ValueExpression::Integer(type_, value, span)) = inner {
- inner = Expression::Value(ValueExpression::Integer(
- type_,
- format_tendril!("-{}", value),
- &op.span + &span,
- ));
+ inner = Expression::Value(ValueExpression::Integer(type_, format!("-{}", value), &op.span + &span));
continue;
} else if let Expression::Value(ValueExpression::Implicit(value, span)) = inner {
- inner = Expression::Value(ValueExpression::Implicit(
- format_tendril!("-{}", value),
- &op.span + &span,
- ));
+ inner = Expression::Value(ValueExpression::Implicit(format!("-{}", value), &op.span + &span));
continue;
}
}
diff --git a/compiler/parser/src/parser/mod.rs b/compiler/parser/src/parser/mod.rs
index adcd32e72f..89020f61b6 100644
--- a/compiler/parser/src/parser/mod.rs
+++ b/compiler/parser/src/parser/mod.rs
@@ -51,14 +51,14 @@ pub(crate) fn assert_no_whitespace(left_span: &Span, right_span: &Span, left: &s
/// Creates a new program from a given file path and source code text.
pub fn parse(handler: &Handler, path: &str, source: &str) -> Result {
- let mut tokens = ParserContext::new(handler, crate::tokenize(path, source.into())?);
+ let mut tokens = ParserContext::new(handler, crate::tokenize(path, source)?);
tokens.parse_program()
}
/// Parses an input file at the given file `path` and `source` code text.
pub fn parse_input(handler: &Handler, path: &str, source: &str) -> Result {
- let mut tokens = ParserContext::new(handler, crate::tokenize(path, source.into())?);
+ let mut tokens = ParserContext::new(handler, crate::tokenize(path, source)?);
tokens.parse_input()
}
diff --git a/compiler/parser/src/test.rs b/compiler/parser/src/test.rs
index 07d6100094..12bdf89074 100644
--- a/compiler/parser/src/test.rs
+++ b/compiler/parser/src/test.rs
@@ -35,7 +35,7 @@ impl Namespace for TokenNamespace {
fn run_test(&self, test: Test) -> Result {
create_session_if_not_set_then(|_| {
- tokenizer::tokenize("test", test.content.into())
+ tokenizer::tokenize("test", &test.content)
.map(|tokens| {
Value::String(
tokens
@@ -80,7 +80,7 @@ fn implicit_value_expr() -> Expression {
}
fn tokenize(test: Test) -> Result, String> {
- tokenizer::tokenize("test", test.content.into()).map_err(|x| x.to_string())
+ tokenizer::tokenize("test", &test.content).map_err(|x| x.to_string())
}
fn all_are_comments(tokens: &[SpannedToken]) -> bool {
diff --git a/compiler/parser/src/tokenizer/lexer.rs b/compiler/parser/src/tokenizer/lexer.rs
index d6f73ed1eb..9aae8388d1 100644
--- a/compiler/parser/src/tokenizer/lexer.rs
+++ b/compiler/parser/src/tokenizer/lexer.rs
@@ -21,52 +21,31 @@ use leo_span::{Span, Symbol};
use serde::{Deserialize, Serialize};
use tendril::StrTendril;
-use std::fmt;
-
-///
-/// Returns the length of the given `wanted` string if the string can be eaten, otherwise returns [`None`].
-/// A string can be eaten if its bytes are at the front of the given `input` array.
-///
-fn eat(input: &[u8], wanted: &str) -> Option {
- let wanted = wanted.as_bytes();
- if input.len() < wanted.len() {
- return None;
- }
- if &input[0..wanted.len()] == wanted {
- return Some(wanted.len());
- }
- None
-}
+use std::{fmt, iter::Peekable};
///
/// Returns a new `StrTendril` string if an identifier can be eaten, otherwise returns [`None`].
/// An identifier can be eaten if its bytes are at the front of the given `input_tendril` string.
///
-fn eat_identifier(input_tendril: &StrTendril) -> Option {
- if input_tendril.is_empty() {
- return None;
- }
- let input = input_tendril.as_bytes();
-
- if !input[0].is_ascii_alphabetic() {
- return None;
+fn eat_identifier(input: &mut Peekable>) -> Option {
+ match input.peek() {
+ None => return None,
+ Some(c) if !c.is_ascii_alphabetic() => return None,
+ _ => {}
}
- let mut i = 1usize;
- while i < input.len() {
- if !input[i].is_ascii_alphanumeric() && input[i] != b'_' {
- break;
- }
- i += 1;
+ let mut ident = String::new();
+ while let Some(c) = input.next_if(|c| c.is_ascii_alphabetic()) {
+ ident.push(c);
}
- Some(input_tendril.subtendril(0, i as u32))
+ Some(ident)
}
impl Token {
///
/// Returns a `char` if a character can be eaten, otherwise returns [`None`].
///
- fn eat_char(input_tendril: StrTendril, escaped: bool, hex: bool, unicode: bool) -> Result {
+ fn _eat_char(input_tendril: StrTendril, escaped: bool, hex: bool, unicode: bool) -> Result {
if input_tendril.is_empty() {
return Err(ParserError::lexer_empty_input_tendril().into());
}
@@ -154,34 +133,30 @@ impl Token {
/// Returns a tuple: [(integer length, integer token)] if an integer can be eaten, otherwise returns [`None`].
/// An integer can be eaten if its bytes are at the front of the given `input_tendril` string.
///
- fn eat_integer(input_tendril: &StrTendril) -> Result<(usize, Token)> {
- if input_tendril.is_empty() {
- return Err(ParserError::lexer_empty_input_tendril().into());
- }
- let input = input_tendril.as_bytes();
- if !input[0].is_ascii_digit() {
- return Err(ParserError::lexer_eat_integer_leading_zero(String::from_utf8_lossy(input)).into());
- }
- let mut i = 1;
+ fn eat_integer(lead: char, input: &mut Peekable>) -> Result<(usize, Token)> {
+ let mut int = String::from(lead);
- while i < input.len() {
- if i == 1 && input[0] == b'0' && input[i] == b'x' {
- return Err(ParserError::lexer_hex_number_provided(
- &input_tendril[0..input_tendril.find('\n').unwrap_or(i) + 1],
- )
- .into());
- }
- if !input[i].is_ascii_digit() {
- break;
+ match input.peek() {
+ None => return Err(ParserError::lexer_empty_input_tendril().into()),
+ Some(c) if !c.is_ascii_digit() => return Err(ParserError::lexer_eat_integer_leading_zero(c).into()),
+ _ => {}
+ }
+
+ while let Some(c) = input.next_if(|c| c.is_ascii_digit()) {
+ if c == '0' && matches!(input.peek(), Some('x')) {
+ int.push(c);
+ int.push(input.next().unwrap());
+ return Err(ParserError::lexer_hex_number_provided(int).into());
}
- i += 1;
+ int.push(c);
}
- Ok((i, Token::Int(input_tendril.subtendril(0, i as u32))))
+
+ Ok((int.len(), Token::Int(int)))
}
/// Returns the number of bytes in an utf-8 encoding that starts with this byte.
- fn utf8_byte_count(byte: u8) -> usize {
+ fn _utf8_byte_count(byte: u8) -> usize {
let mut mask = 0x80;
let mut result = 0;
while byte & mask > 0 {
@@ -201,284 +176,155 @@ impl Token {
/// Returns a tuple: [(token length, token)] if the next token can be eaten, otherwise returns [`None`].
/// The next token can be eaten if the bytes at the front of the given `input_tendril` string can be scanned into a token.
///
- pub(crate) fn eat(input_tendril: StrTendril) -> Result<(usize, Token)> {
+ pub(crate) fn eat(input_tendril: &str) -> Result<(usize, Token)> {
if input_tendril.is_empty() {
return Err(ParserError::lexer_empty_input_tendril().into());
}
- let input = input_tendril.as_bytes();
- match input[0] {
- x if x.is_ascii_whitespace() => return Ok((1, Token::WhiteSpace)),
- b'"' => {
- let mut i = 1;
- let mut len = 1;
- let mut start = 1;
- let mut in_escape = false;
- let mut escaped = false;
- let mut hex = false;
- let mut unicode = false;
- let mut end = false;
- let mut string = Vec::new();
- while i < input.len() {
- // Get the length of the utf-8 encoding here
- // and position i at the last byte.
- if input[i] & 0x80 > 0 {
- len = Self::utf8_byte_count(input[i]);
- i += len;
+ let mut input = input_tendril.chars().peekable();
- if unicode {
- return Err(
- ParserError::lexer_emoji_inside_escaped_unicode_char(&input_tendril[0..i]).into(),
- );
- }
-
- continue;
- }
-
- if !in_escape {
- if input[i] == b'"' {
- end = true;
- break;
- } else if input[i] == b'\\' {
- in_escape = true;
- start = i;
- i += 1;
- continue;
- }
- } else {
- len += 1;
-
- match input[i] {
- b'x' => {
- hex = true;
- }
- b'u' => {
- unicode = true;
- }
- b'}' if unicode => {
- in_escape = false;
- }
- _ if !hex && !unicode => {
- escaped = true;
- in_escape = false;
- }
- _ if hex && len == 4 => {
- in_escape = false;
- }
- _ => {}
- }
- }
-
- if !in_escape {
- let character = Self::eat_char(
- input_tendril.subtendril(start as u32, len as u32),
- escaped,
- hex,
- unicode,
- )?;
- len = 1;
- escaped = false;
- hex = false;
- unicode = false;
- string.push(character.into());
- }
-
- i += 1;
-
- if !escaped && !hex && !unicode {
- start = i;
- }
- }
-
- if i == input.len() || !end {
- return Err(ParserError::lexer_string_not_closed(String::from_utf8_lossy(&input[0..i])).into());
- }
-
- return Ok((i + 1, Token::StringLit(string)));
+ match input.next() {
+ Some(x) if x.is_ascii_whitespace() => return Ok((1, Token::WhiteSpace)),
+ Some(lead) if lead.is_ascii_digit() => {
+ return Self::eat_integer(lead, &mut input);
}
- b'\'' => {
- let mut i = 1;
- let mut in_escape = false;
- let mut escaped = false;
- let mut hex = false;
- let mut escaped_unicode = false;
- let mut unicode_char = false;
- let mut end = false;
-
- while i < input.len() {
- if input[i] & 0x80 > 0 && !unicode_char {
- i += Self::utf8_byte_count(input[i]);
- unicode_char = true;
- continue;
- } else if input[i] & 0x80 > 0 && unicode_char {
- i += Self::utf8_byte_count(input[i]);
- return Err(ParserError::lexer_invalid_char(&input_tendril[0..i]).into());
- } else if !in_escape || unicode_char {
- if input[i] == b'\'' {
- end = true;
- break;
- } else if unicode_char {
- return Err(ParserError::lexer_invalid_char(
- // grab the contents of everything between the '' if possible.
- // else just show the character right before stuff went wrong.
- &input_tendril[0..input_tendril[1..].find('\'').unwrap_or(i - 1) + 1],
- )
- .into());
- } else if input[i] == b'\\' {
- in_escape = true;
- }
- } else {
- if input[i] == b'x' {
- hex = true;
- } else if input[i] == b'u' {
- let one_ahead = input.get(i + 1);
- if matches!(one_ahead, Some(b'{')) {
- escaped_unicode = true;
- } else if one_ahead.is_some() {
- return Err(ParserError::lexer_expected_valid_escaped_char(input[i + 1]).into());
- } else {
- return Err(ParserError::lexer_expected_valid_escaped_char(input[i]).into());
- }
- } else {
- escaped = true;
- }
-
- in_escape = false;
- }
-
- i += 1;
- }
-
- if !end {
- return Err(ParserError::lexer_char_not_closed(String::from_utf8_lossy(&input[0..i])).into());
- }
-
- let character = Self::eat_char(
- input_tendril.subtendril(1, (i - 1) as u32),
- escaped,
- hex,
- escaped_unicode,
- )?;
- return Ok((i + 1, Token::CharLit(character)));
- }
- x if x.is_ascii_digit() => {
- return Self::eat_integer(&input_tendril);
- }
- b'!' => {
- if let Some(len) = eat(input, "!=") {
- return Ok((len, Token::NotEq));
+ Some('!') => {
+ if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::NotEq));
}
return Ok((1, Token::Not));
}
- b'?' => {
+ Some('?') => {
return Ok((1, Token::Question));
}
- b'&' => {
- if let Some(len) = eat(input, "&&") {
- return Ok((len, Token::And));
+ Some('&') => {
+ if input.next_if_eq(&'&').is_some() {
+ return Ok((2, Token::And));
}
return Ok((1, Token::Ampersand));
}
- b'(' => return Ok((1, Token::LeftParen)),
- b')' => return Ok((1, Token::RightParen)),
- b'_' => return Ok((1, Token::Underscore)),
- b'*' => {
- if let Some(len) = eat(input, "**") {
- if let Some(inner_len) = eat(&input[len..], "=") {
- return Ok((len + inner_len, Token::ExpEq));
+ Some('(') => return Ok((1, Token::LeftParen)),
+ Some(')') => return Ok((1, Token::RightParen)),
+ Some('_') => return Ok((1, Token::Underscore)),
+ Some('*') => {
+ if input.next_if_eq(&'*').is_some() {
+ if input.next_if_eq(&'=').is_some() {
+ return Ok((3, Token::ExpEq));
}
- return Ok((len, Token::Exp));
- } else if let Some(len) = eat(input, "*=") {
- return Ok((len, Token::MulEq));
+ return Ok((2, Token::Exp));
+ } else if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::MulEq));
}
return Ok((1, Token::Mul));
}
- b'+' => {
- if let Some(len) = eat(input, "+=") {
- return Ok((len, Token::AddEq));
+ Some('+') => {
+ if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::AddEq));
}
return Ok((1, Token::Add));
}
- b',' => return Ok((1, Token::Comma)),
- b'-' => {
- if let Some(len) = eat(input, "->") {
- return Ok((len, Token::Arrow));
- } else if let Some(len) = eat(input, "-=") {
- return Ok((len, Token::MinusEq));
+ Some(',') => return Ok((1, Token::Comma)),
+ Some('-') => {
+ if input.next_if_eq(&'>').is_some() {
+ return Ok((2, Token::Arrow));
+ } else if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::MinusEq));
}
return Ok((1, Token::Minus));
}
- b'.' => {
- if let Some(len) = eat(input, "...") {
- return Ok((len, Token::DotDotDot));
- } else if let Some(len) = eat(input, "..") {
- return Ok((len, Token::DotDot));
+ Some('.') => {
+ if input.next_if_eq(&'.').is_some() {
+ if input.next_if_eq(&'.').is_some() {
+ return Ok((3, Token::DotDotDot));
+ } else {
+ return Ok((2, Token::DotDot));
+ }
}
return Ok((1, Token::Dot));
}
- b'/' => {
- if eat(input, "//").is_some() {
- let eol = input.iter().position(|x| *x == b'\n');
- let len = if let Some(eol) = eol { eol + 1 } else { input.len() };
- return Ok((len, Token::CommentLine(input_tendril.subtendril(0, len as u32))));
- } else if eat(input, "/*").is_some() {
- if input.is_empty() {
+ Some(c) if c == '/' => {
+ let mut comment = String::from(c);
+ if let Some(c) = input.next_if_eq(&'/') {
+ comment.push(c);
+
+ while let Some(c) = input.next_if(|c| c != &'\n') {
+ comment.push(c);
+ }
+
+ if input.next_if_eq(&'\n').is_some() {
+ return Ok((comment.len() + 1, Token::CommentLine(comment)));
+ }
+
+ return Ok((comment.len(), Token::CommentLine(comment)));
+ } else if let Some(c) = input.next_if_eq(&'*') {
+ comment.push(c);
+
+ if input.peek().is_none() {
return Err(ParserError::lexer_empty_block_comment().into());
}
- let eol = input.windows(2).skip(2).position(|x| x[0] == b'*' && x[1] == b'/');
- let len = if let Some(eol) = eol {
- eol + 4
- } else {
- return Err(ParserError::lexer_block_comment_does_not_close_before_eof(
- String::from_utf8_lossy(&input[0..]),
- )
- .into());
- };
- return Ok((len, Token::CommentBlock(input_tendril.subtendril(0, len as u32))));
- } else if let Some(len) = eat(input, "/=") {
- return Ok((len, Token::DivEq));
+
+ let mut ended = false;
+ while let Some(c) = input.next() {
+ comment.push(c);
+ if c == '*' && input.next_if_eq(&'/').is_some() {
+ comment.push('/');
+ ended = true;
+ break;
+ }
+ }
+
+ if !ended {
+ return Err(ParserError::lexer_block_comment_does_not_close_before_eof(comment).into());
+ }
+ return Ok((comment.len() + 4, Token::CommentBlock(comment)));
+ } else if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::DivEq));
}
return Ok((1, Token::Div));
}
- b':' => {
- if let Some(len) = eat(input, "::") {
- return Ok((len, Token::DoubleColon));
+ Some(':') => {
+ if input.next_if_eq(&':').is_some() {
+ return Ok((2, Token::DoubleColon));
} else {
return Ok((1, Token::Colon));
}
}
- b';' => return Ok((1, Token::Semicolon)),
- b'<' => {
- if let Some(len) = eat(input, "<=") {
- return Ok((len, Token::LtEq));
+ Some(';') => return Ok((1, Token::Semicolon)),
+ Some('<') => {
+ if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::LtEq));
}
return Ok((1, Token::Lt));
}
- b'>' => {
- if let Some(len) = eat(input, ">=") {
- return Ok((len, Token::GtEq));
+ Some('>') => {
+ if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::GtEq));
}
return Ok((1, Token::Gt));
}
- b'=' => {
- if let Some(len) = eat(input, "==") {
- return Ok((len, Token::Eq));
+ Some('=') => {
+ if input.next_if_eq(&'=').is_some() {
+ return Ok((2, Token::Eq));
}
return Ok((1, Token::Assign));
}
- b'@' => return Ok((1, Token::At)),
- b'[' => return Ok((1, Token::LeftSquare)),
- b']' => return Ok((1, Token::RightSquare)),
- b'{' => return Ok((1, Token::LeftCurly)),
- b'}' => return Ok((1, Token::RightCurly)),
- b'|' => {
- if let Some(len) = eat(input, "||") {
- return Ok((len, Token::Or));
+ Some('@') => return Ok((1, Token::At)),
+ Some('[') => return Ok((1, Token::LeftSquare)),
+ Some(']') => return Ok((1, Token::RightSquare)),
+ Some('{') => return Ok((1, Token::LeftCurly)),
+ Some('}') => return Ok((1, Token::RightCurly)),
+ Some('|') => {
+ if input.next_if_eq(&'|').is_some() {
+ return Ok((2, Token::Or));
+ } else if let Some(found) = input.next() {
+ return Err(ParserError::lexer_expected_but_found(found, '|').into());
+ } else {
+ return Err(ParserError::lexer_empty_input_tendril().into());
}
}
_ => (),
}
- if let Some(ident) = eat_identifier(&input_tendril) {
+ if let Some(ident) = eat_identifier(&mut input) {
return Ok((
ident.len(),
match &*ident {
@@ -523,7 +369,7 @@ impl Token {
));
}
- Err(ParserError::could_not_lex(String::from_utf8_lossy(&input[0..])).into())
+ Err(ParserError::could_not_lex(input.collect::()).into())
}
}
diff --git a/compiler/parser/src/tokenizer/mod.rs b/compiler/parser/src/tokenizer/mod.rs
index 81d674b0b1..ec76528322 100644
--- a/compiler/parser/src/tokenizer/mod.rs
+++ b/compiler/parser/src/tokenizer/mod.rs
@@ -31,17 +31,15 @@ pub(crate) use self::lexer::*;
use leo_errors::{ParserError, Result};
use leo_span::Span;
-use tendril::StrTendril;
-
/// Creates a new vector of spanned tokens from a given file path and source code text.
-pub(crate) fn tokenize(path: &str, input: StrTendril) -> Result> {
+pub(crate) fn tokenize(path: &str, input: &str) -> Result> {
let path = Arc::new(path.to_string());
let mut tokens = vec![];
let mut index = 0usize;
let mut line_no = 1usize;
let mut line_start = 0usize;
while input.len() > index {
- match Token::eat(input.subtendril(index as u32, (input.len() - index) as u32))? {
+ match Token::eat(&input[index..(input.len() - index)])? {
(token_len, Token::WhiteSpace) => {
if token_len == 0 && index == input.len() {
break;
@@ -54,10 +52,7 @@ pub(crate) fn tokenize(path: &str, input: StrTendril) -> Result Result {
diff --git a/compiler/parser/src/tokenizer/token.rs b/compiler/parser/src/tokenizer/token.rs
index 3dca55e1cf..22afe2635d 100644
--- a/compiler/parser/src/tokenizer/token.rs
+++ b/compiler/parser/src/tokenizer/token.rs
@@ -18,7 +18,6 @@ use leo_span::{sym, Symbol};
use serde::{Deserialize, Serialize};
use std::fmt;
-use tendril::StrTendril;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Char {
@@ -50,14 +49,14 @@ impl fmt::Display for Char {
pub enum Token {
// Lexical Grammar
// Literals
- CommentLine(#[serde(with = "leo_span::tendril_json")] StrTendril),
- CommentBlock(#[serde(with = "leo_span::tendril_json")] StrTendril),
+ CommentLine(String),
+ CommentBlock(String),
StringLit(Vec),
Ident(Symbol),
- Int(#[serde(with = "leo_span::tendril_json")] StrTendril),
+ Int(String),
True,
False,
- AddressLit(#[serde(with = "leo_span::tendril_json")] StrTendril),
+ AddressLit(String),
CharLit(Char),
WhiteSpace,
diff --git a/leo/errors/src/parser/parser_errors.rs b/leo/errors/src/parser/parser_errors.rs
index 08a5905e44..9b02ce6979 100644
--- a/leo/errors/src/parser/parser_errors.rs
+++ b/leo/errors/src/parser/parser_errors.rs
@@ -398,4 +398,13 @@ create_errors!(
msg: "do not put parens around single dimension array size",
help: None,
}
+
+ /// For when a user puts parens around a single defined variable.
+ @backtraced
+ lexer_expected_but_found {
+ args: (found: impl Display, expected: impl Display),
+ msg: format!("Found the char `{}`, but expected `{}`", found, expected),
+ help: None,
+ }
+
);
diff --git a/leo/span/src/span.rs b/leo/span/src/span.rs
index 50f2d32080..912d2203e9 100644
--- a/leo/span/src/span.rs
+++ b/leo/span/src/span.rs
@@ -20,7 +20,6 @@ use std::{fmt, sync::Arc, usize};
use serde::ser::{Serialize, SerializeStruct, Serializer};
use serde::Deserialize;
-use tendril::StrTendril;
/// The span type which tracks where formatted errors originate from in a Leo file.
/// This is used in many spots throughout the rest of the Leo crates.
@@ -36,9 +35,8 @@ pub struct Span {
pub col_stop: usize,
/// The path to the Leo file containing the error.
pub path: Arc,
- #[serde(with = "crate::tendril_json")]
/// The content of the line(s) that the span is found on.
- pub content: StrTendril,
+ pub content: String,
}
impl Span {
@@ -55,7 +53,7 @@ impl Span {
col_start: usize,
col_stop: usize,
path: Arc,
- content: StrTendril,
+ content: String,
) -> Self {
Self {
line_start,
@@ -156,7 +154,7 @@ impl std::ops::Add for Span {
new_content.push(format!("{:<1$}...", " ", other.col_start + 4));
}
}
- let new_content = new_content.join("\n").into();
+ let new_content = new_content.join("\n");
if self.line_start < other.line_stop {
Span {
line_start: self.line_start,
From 5034294d09e54342524d7f8ca8650597cb22c474 Mon Sep 17 00:00:00 2001
From: gluax <16431709+gluax@users.noreply.github.com>
Date: Tue, 15 Mar 2022 17:13:02 -0700
Subject: [PATCH 02/13] tokenizing almost working
---
compiler/parser/src/test.rs | 1 -
compiler/parser/src/tokenizer/lexer.rs | 126 +++++++++---
compiler/parser/src/tokenizer/mod.rs | 24 ++-
compiler/parser/src/tokenizer/token.rs | 2 +-
compiler/parser/tests/serialization/json.rs | 217 --------------------
leo/errors/src/parser/parser_errors.rs | 4 +-
6 files changed, 124 insertions(+), 250 deletions(-)
delete mode 100644 compiler/parser/tests/serialization/json.rs
diff --git a/compiler/parser/src/test.rs b/compiler/parser/src/test.rs
index 12bdf89074..3e91f5d7f9 100644
--- a/compiler/parser/src/test.rs
+++ b/compiler/parser/src/test.rs
@@ -259,7 +259,6 @@ impl Runner for TestRunner {
}
}
-#[test]
pub fn parser_tests() {
leo_test_framework::run_tests(&TestRunner, "parser");
}
diff --git a/compiler/parser/src/tokenizer/lexer.rs b/compiler/parser/src/tokenizer/lexer.rs
index 9aae8388d1..ded4dc2aef 100644
--- a/compiler/parser/src/tokenizer/lexer.rs
+++ b/compiler/parser/src/tokenizer/lexer.rs
@@ -35,7 +35,7 @@ fn eat_identifier(input: &mut Peekable>) -> Option>) -> Result<(usize, Token)> {
- let mut int = String::from(lead);
-
- match input.peek() {
- None => return Err(ParserError::lexer_empty_input_tendril().into()),
- Some(c) if !c.is_ascii_digit() => return Err(ParserError::lexer_eat_integer_leading_zero(c).into()),
- _ => {}
+ fn eat_integer(input: &mut Peekable>) -> Result<(usize, Token)> {
+ if input.peek().is_none() {
+ return Err(ParserError::lexer_empty_input_tendril().into());
}
+ let mut int = String::new();
while let Some(c) = input.next_if(|c| c.is_ascii_digit()) {
if c == '0' && matches!(input.peek(), Some('x')) {
int.push(c);
@@ -183,30 +180,78 @@ impl Token {
let mut input = input_tendril.chars().peekable();
- match input.next() {
- Some(x) if x.is_ascii_whitespace() => return Ok((1, Token::WhiteSpace)),
- Some(lead) if lead.is_ascii_digit() => {
- return Self::eat_integer(lead, &mut input);
+ match input.peek() {
+ Some(x) if x.is_ascii_whitespace() => {
+ input.next();
+ return Ok((1, Token::WhiteSpace));
+ }
+ Some('"') => {
+ let mut string = Vec::new();
+ input.next();
+
+ while let Some(c) = input.next_if(|c| c != &'"') {
+ let character = leo_ast::Char::Scalar(c);
+ string.push(character);
+ }
+
+ if input.next_if_eq(&'"').is_some() {
+ return Ok((string.len() + 2, Token::StringLit(string)));
+ }
+
+ return Err(ParserError::lexer_string_not_closed(string).into());
+ }
+ Some('\'') => {
+ input.next();
+
+ if let Some(c) = input.next() {
+ dbg!(&c);
+ if input.next_if_eq(&'\'').is_some() {
+ input.next();
+ return Ok((c.len_utf8() + 2, Token::CharLit(Char::Scalar(c))));
+ } else if let Some(c) = input.next() {
+ return Err(ParserError::lexer_string_not_closed(c).into());
+ } else {
+ return Err(ParserError::lexer_empty_input_tendril().into());
+ }
+ }
+
+ return Err(ParserError::lexer_empty_input_tendril().into());
+ }
+ Some(x) if x.is_ascii_digit() => {
+ return Self::eat_integer(&mut input);
}
Some('!') => {
+ input.next();
if input.next_if_eq(&'=').is_some() {
return Ok((2, Token::NotEq));
}
return Ok((1, Token::Not));
}
Some('?') => {
+ input.next();
return Ok((1, Token::Question));
}
Some('&') => {
+ input.next();
if input.next_if_eq(&'&').is_some() {
return Ok((2, Token::And));
}
return Ok((1, Token::Ampersand));
}
- Some('(') => return Ok((1, Token::LeftParen)),
- Some(')') => return Ok((1, Token::RightParen)),
- Some('_') => return Ok((1, Token::Underscore)),
+ Some('(') => {
+ input.next();
+ return Ok((1, Token::LeftParen));
+ }
+ Some(')') => {
+ input.next();
+ return Ok((1, Token::RightParen));
+ }
+ Some('_') => {
+ input.next();
+ return Ok((1, Token::Underscore));
+ }
Some('*') => {
+ input.next();
if input.next_if_eq(&'*').is_some() {
if input.next_if_eq(&'=').is_some() {
return Ok((3, Token::ExpEq));
@@ -218,13 +263,18 @@ impl Token {
return Ok((1, Token::Mul));
}
Some('+') => {
+ input.next();
if input.next_if_eq(&'=').is_some() {
return Ok((2, Token::AddEq));
}
return Ok((1, Token::Add));
}
- Some(',') => return Ok((1, Token::Comma)),
+ Some(',') => {
+ input.next();
+ return Ok((1, Token::Comma));
+ }
Some('-') => {
+ input.next();
if input.next_if_eq(&'>').is_some() {
return Ok((2, Token::Arrow));
} else if input.next_if_eq(&'=').is_some() {
@@ -233,6 +283,7 @@ impl Token {
return Ok((1, Token::Minus));
}
Some('.') => {
+ input.next();
if input.next_if_eq(&'.').is_some() {
if input.next_if_eq(&'.').is_some() {
return Ok((3, Token::DotDotDot));
@@ -242,8 +293,9 @@ impl Token {
}
return Ok((1, Token::Dot));
}
- Some(c) if c == '/' => {
- let mut comment = String::from(c);
+ Some(c) if c == &'/' => {
+ let mut comment = String::from(*c);
+ input.next();
if let Some(c) = input.next_if_eq(&'/') {
comment.push(c);
@@ -251,7 +303,8 @@ impl Token {
comment.push(c);
}
- if input.next_if_eq(&'\n').is_some() {
+ if let Some(newline) = input.next_if_eq(&'\n') {
+ comment.push(newline);
return Ok((comment.len() + 1, Token::CommentLine(comment)));
}
@@ -283,37 +336,60 @@ impl Token {
return Ok((1, Token::Div));
}
Some(':') => {
+ input.next();
if input.next_if_eq(&':').is_some() {
return Ok((2, Token::DoubleColon));
} else {
return Ok((1, Token::Colon));
}
}
- Some(';') => return Ok((1, Token::Semicolon)),
+ Some(';') => {
+ input.next();
+ return Ok((1, Token::Semicolon));
+ }
Some('<') => {
+ input.next();
if input.next_if_eq(&'=').is_some() {
return Ok((2, Token::LtEq));
}
return Ok((1, Token::Lt));
}
Some('>') => {
+ input.next();
if input.next_if_eq(&'=').is_some() {
return Ok((2, Token::GtEq));
}
return Ok((1, Token::Gt));
}
Some('=') => {
+ input.next();
if input.next_if_eq(&'=').is_some() {
return Ok((2, Token::Eq));
}
return Ok((1, Token::Assign));
}
- Some('@') => return Ok((1, Token::At)),
- Some('[') => return Ok((1, Token::LeftSquare)),
- Some(']') => return Ok((1, Token::RightSquare)),
- Some('{') => return Ok((1, Token::LeftCurly)),
- Some('}') => return Ok((1, Token::RightCurly)),
+ Some('@') => {
+ input.next();
+ return Ok((1, Token::At));
+ }
+ Some('[') => {
+ input.next();
+ return Ok((1, Token::LeftSquare));
+ }
+ Some(']') => {
+ input.next();
+ return Ok((1, Token::RightSquare));
+ }
+ Some('{') => {
+ input.next();
+ return Ok((1, Token::LeftCurly));
+ }
+ Some('}') => {
+ input.next();
+ return Ok((1, Token::RightCurly));
+ }
Some('|') => {
+ input.next();
if input.next_if_eq(&'|').is_some() {
return Ok((2, Token::Or));
} else if let Some(found) = input.next() {
diff --git a/compiler/parser/src/tokenizer/mod.rs b/compiler/parser/src/tokenizer/mod.rs
index ec76528322..919c883fd6 100644
--- a/compiler/parser/src/tokenizer/mod.rs
+++ b/compiler/parser/src/tokenizer/mod.rs
@@ -39,7 +39,7 @@ pub(crate) fn tokenize(path: &str, input: &str) -> Result> {
let mut line_no = 1usize;
let mut line_start = 0usize;
while input.len() > index {
- match Token::eat(&input[index..(input.len() - index)])? {
+ match Token::eat(&input[index..input.len()])? {
(token_len, Token::WhiteSpace) => {
if token_len == 0 && index == input.len() {
break;
@@ -52,7 +52,12 @@ pub(crate) fn tokenize(path: &str, input: &str) -> Result> {
index - line_start + 1,
index - line_start + 2,
path,
- input[line_start..input[line_start..].find('\n').unwrap_or(input.len())].to_string(),
+ input[line_start
+ ..input[line_start..]
+ .find('\n')
+ .map(|i| i + line_start)
+ .unwrap_or(input.len())]
+ .to_string(),
),
)
.into());
@@ -78,7 +83,12 @@ pub(crate) fn tokenize(path: &str, input: &str) -> Result> {
index - line_start + 1,
index - line_start + token_len + 1,
path.clone(),
- input[line_start..input[line_start..].find('\n').unwrap_or(input.len() - line_start)].to_string(),
+ input[line_start
+ ..input[line_start..]
+ .find('\n')
+ .map(|i| i + line_start)
+ .unwrap_or(input.len())]
+ .to_string(),
);
match &token {
Token::CommentLine(_) => {
@@ -121,6 +131,12 @@ mod tests {
let tokens = tokenize(
"test_path",
r#"
+ 'a'
+ '😭'
+ '\u{10001F}'
+ '\x7f'
+ '\x00'
+ '\x37'
"test"
"test{}test"
"test{}"
@@ -219,7 +235,7 @@ mod tests {
assert_eq!(
output,
- r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut & return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
+ r#"'a' '😭' '\u{10001F}' "test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut & return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
/* test */ // "#
);
});
diff --git a/compiler/parser/src/tokenizer/token.rs b/compiler/parser/src/tokenizer/token.rs
index 22afe2635d..a9c3e17c86 100644
--- a/compiler/parser/src/tokenizer/token.rs
+++ b/compiler/parser/src/tokenizer/token.rs
@@ -258,7 +258,7 @@ impl fmt::Display for Token {
True => write!(f, "true"),
False => write!(f, "false"),
AddressLit(s) => write!(f, "{}", s),
- CharLit(s) => write!(f, "{}", s),
+ CharLit(s) => write!(f, "'{}'", s),
WhiteSpace => write!(f, "whitespace"),
At => write!(f, "@"),
diff --git a/compiler/parser/tests/serialization/json.rs b/compiler/parser/tests/serialization/json.rs
deleted file mode 100644
index b0fcd85eef..0000000000
--- a/compiler/parser/tests/serialization/json.rs
+++ /dev/null
@@ -1,217 +0,0 @@
-// Copyright (C) 2019-2022 Aleo Systems Inc.
-// This file is part of the Leo library.
-
-// The Leo library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// The Leo library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with the Leo library. If not, see .
-
-use leo_ast::Ast;
-#[cfg(not(feature = "ci_skip"))]
-use leo_ast::Program;
-use leo_errors::{emitter::Handler, LeoError, Result};
-
-use std::fs::File;
-use std::io::BufReader;
-use std::iter::Iterator;
-use std::path::{Path, PathBuf};
-
-fn to_ast(program_filepath: &Path) -> Result {
- let program_string = std::fs::read_to_string(program_filepath).expect("failed to open test");
-
- // Parses the Leo file and constructs a leo ast.
- leo_parser::parse_ast(&Handler::default(), "", &program_string)
-}
-
-fn setup() {
- std::env::set_var("LEO_TESTFRAMEWORK", "true");
-}
-
-fn clean() {
- std::env::remove_var("LEO_TESTFRAMEWORK");
-}
-
-#[test]
-#[cfg(not(feature = "ci_skip"))]
-fn test_serialize() {
- setup();
-
- // Construct an ast from the given test file.
- let ast = {
- let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- program_filepath.push("tests/serialization/leo/one_plus_one.leo");
-
- to_ast(&program_filepath).unwrap()
- };
-
- // Serializes the ast into JSON format.
- let serialized_ast: Program = serde_json::from_value(serde_json::to_value(ast.as_repr()).unwrap()).unwrap();
-
- // Load the expected ast.
- let expected: Program = serde_json::from_str(include_str!("./expected_leo_ast/one_plus_one.json")).unwrap();
-
- clean();
- assert_eq!(expected, serialized_ast);
-}
-
-#[test]
-#[cfg(not(feature = "ci_skip"))]
-fn test_serialize_no_span() {
- setup();
-
- let program_paths = vec![
- "tests/serialization/leo/linear_regression.leo",
- "tests/serialization/leo/palindrome.leo",
- "tests/serialization/leo/pedersen_hash.leo",
- "tests/serialization/leo/silly_sudoku.leo",
- ];
-
- let json_paths = vec![
- "tests/serialization/expected_leo_ast/linear_regression.json",
- "tests/serialization/expected_leo_ast/palindrome.json",
- "tests/serialization/expected_leo_ast/pedersen_hash.json",
- "tests/serialization/expected_leo_ast/silly_sudoku.json",
- ];
-
- for (program_path, json_path) in program_paths.into_iter().zip(json_paths) {
- // Construct an ast from the given test file.
- let ast = {
- let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- program_filepath.push(program_path);
- to_ast(&program_filepath).unwrap()
- };
-
- let json_reader = {
- let mut json_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- json_filepath.push(json_path);
- let file = File::open(json_filepath).expect("Failed to read expected ast file");
- BufReader::new(file)
- };
-
- // Serializes the ast into JSON format.
- let mut serialized_ast: serde_json::Value = serde_json::to_value(ast.as_repr()).unwrap();
- remove_key_from_json(&mut serialized_ast, "span");
- serialized_ast = normalize_json_value(serialized_ast);
-
- // Load the expected ast.
- let expected: serde_json::Value = serde_json::from_reader(json_reader).unwrap();
-
- assert_eq!(expected, serialized_ast);
- }
- clean();
-}
-
-// Helper functions to recursively filter keys from AST JSON.
-// Redeclaring here since we don't want to make this public.
-fn remove_key_from_json(value: &mut serde_json::Value, key: &str) {
- match value {
- serde_json::value::Value::Object(map) => {
- map.remove(key);
- for val in map.values_mut() {
- remove_key_from_json(val, key);
- }
- }
- serde_json::value::Value::Array(values) => {
- for val in values.iter_mut() {
- remove_key_from_json(val, key);
- }
- }
- _ => (),
- }
-}
-
-// Helper function to normalize AST
-// Redeclaring here because we don't want to make this public
-fn normalize_json_value(value: serde_json::Value) -> serde_json::Value {
- match value {
- serde_json::Value::Array(vec) => {
- let orig_length = vec.len();
- let mut new_vec: Vec = vec
- .into_iter()
- .filter(|v| !matches!(v, serde_json::Value::Object(map) if map.is_empty()))
- .map(normalize_json_value)
- .collect();
-
- if orig_length == 2 && new_vec.len() == 1 {
- new_vec.pop().unwrap()
- } else {
- serde_json::Value::Array(new_vec)
- }
- }
- serde_json::Value::Object(map) => {
- serde_json::Value::Object(map.into_iter().map(|(k, v)| (k, normalize_json_value(v))).collect())
- }
- _ => value,
- }
-}
-
-// TODO Renable when we don't write spans to snapshots.
-/* #[test]
-#[cfg(not(feature = "ci_skip"))]
-fn test_deserialize() {
- setup();
-
- // Load the expected ast.
- let expected_ast = {
- let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- program_filepath.push("tests/serialization/main.leo");
-
- to_ast(&program_filepath).unwrap()
- };
-
- // Construct an ast by deserializing a ast JSON file.
- let serialized_ast = include_str!("expected_leo_ast.json");
- let ast = Ast::from_json_string(serialized_ast).unwrap();
-
- clean();
- assert_eq!(expected_ast, ast);
-}
-
-#[test]
-fn test_serialize_deserialize_serialize() {
- setup();
-
- // Construct an ast from the given test file.
- let ast = {
- let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- program_filepath.push("tests/serialization/main.leo");
-
- to_ast(&program_filepath).unwrap()
- };
-
- // Serializes the ast into JSON format.
- let serialized_ast = ast.to_json_string().unwrap();
-
- // Deserializes the serialized ast into an ast.
- let ast = Ast::from_json_string(&serialized_ast).unwrap();
-
- // Reserializes the ast into JSON format.
- let reserialized_ast = ast.to_json_string().unwrap();
-
- clean();
- assert_eq!(serialized_ast, reserialized_ast);
-} */
-
-#[test]
-fn test_generic_parser_error() {
- setup();
-
- let error_result = {
- let mut program_filepath = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
- program_filepath.push("tests/serialization/leo/parser_error.leo");
-
- to_ast(&program_filepath)
- }
- .map_err(|err| matches!(err, LeoError::ParserError(_)));
-
- clean();
- assert!(error_result.err().unwrap());
-}
diff --git a/leo/errors/src/parser/parser_errors.rs b/leo/errors/src/parser/parser_errors.rs
index 9b02ce6979..215cfd51f9 100644
--- a/leo/errors/src/parser/parser_errors.rs
+++ b/leo/errors/src/parser/parser_errors.rs
@@ -251,8 +251,8 @@ create_errors!(
/// When a string is not properly closed.
@backtraced
lexer_string_not_closed {
- args: (input: impl Display),
- msg: format!("Expected a closed string but found `{}`.", input),
+ args: (input: impl Debug),
+ msg: format!("Expected a closed string but found `{:?}`.", input),
help: None,
}
From 9391a3156989463a1cd613ae109679eade1e71b1 Mon Sep 17 00:00:00 2001
From: gluax <16431709+gluax@users.noreply.github.com>
Date: Wed, 16 Mar 2022 12:27:11 -0700
Subject: [PATCH 03/13] realized raw strings don't play nicely for testing
strings
---
compiler/ast/src/chars/char_value.rs | 2 +-
compiler/parser/src/test.rs | 1 +
compiler/parser/src/tokenizer/lexer.rs | 191 ++++++++++++++-----------
compiler/parser/src/tokenizer/mod.rs | 7 +-
compiler/parser/src/tokenizer/token.rs | 2 +-
5 files changed, 109 insertions(+), 94 deletions(-)
diff --git a/compiler/ast/src/chars/char_value.rs b/compiler/ast/src/chars/char_value.rs
index 37857a3c5d..37b6fbd87f 100644
--- a/compiler/ast/src/chars/char_value.rs
+++ b/compiler/ast/src/chars/char_value.rs
@@ -48,7 +48,7 @@ impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Scalar(c) => write!(f, "{}", c),
- Self::NonScalar(c) => write!(f, "{}", c),
+ Self::NonScalar(c) => write!(f, "{:X}", c),
}
}
}
diff --git a/compiler/parser/src/test.rs b/compiler/parser/src/test.rs
index 3e91f5d7f9..12bdf89074 100644
--- a/compiler/parser/src/test.rs
+++ b/compiler/parser/src/test.rs
@@ -259,6 +259,7 @@ impl Runner for TestRunner {
}
}
+#[test]
pub fn parser_tests() {
leo_test_framework::run_tests(&TestRunner, "parser");
}
diff --git a/compiler/parser/src/tokenizer/lexer.rs b/compiler/parser/src/tokenizer/lexer.rs
index ded4dc2aef..db762f4663 100644
--- a/compiler/parser/src/tokenizer/lexer.rs
+++ b/compiler/parser/src/tokenizer/lexer.rs
@@ -19,7 +19,6 @@ use leo_errors::{ParserError, Result};
use leo_span::{Span, Symbol};
use serde::{Deserialize, Serialize};
-use tendril::StrTendril;
use std::{fmt, iter::Peekable};
@@ -42,91 +41,109 @@ fn eat_identifier(input: &mut Peekable>) -> Option Result {
- if input_tendril.is_empty() {
+ fn eat_unicode_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ let mut unicode = String::new();
+ // Account for the chars '\' and 'u'.
+ let mut len = 2;
+
+ if input.next_if_eq(&'{').is_some() {
+ len += 1;
+ } else if let Some(c) = input.next() {
+ return Err(ParserError::lexer_unopened_escaped_unicode_char(c).into());
+ } else {
return Err(ParserError::lexer_empty_input_tendril().into());
}
- if escaped {
- let string = input_tendril.to_string();
- let escaped = &string[1..input_tendril.len()];
+ while let Some(c) = input.next_if(|c| c != &'}') {
+ len += 1;
+ unicode.push(c);
+ }
- if escaped.len() != 1 {
- return Err(ParserError::lexer_escaped_char_incorrect_length(escaped).into());
- }
+ if input.next_if_eq(&'}').is_some() {
+ len += 1;
+ } else {
+ return Err(ParserError::lexer_unclosed_escaped_unicode_char(unicode).into());
+ }
- if let Some(character) = escaped.chars().next() {
- return match character {
- '0' => Ok(Char::Scalar(0 as char)),
- 't' => Ok(Char::Scalar(9 as char)),
- 'n' => Ok(Char::Scalar(10 as char)),
- 'r' => Ok(Char::Scalar(13 as char)),
- '\"' => Ok(Char::Scalar(34 as char)),
- '\'' => Ok(Char::Scalar(39 as char)),
- '\\' => Ok(Char::Scalar(92 as char)),
- _ => return Err(ParserError::lexer_expected_valid_escaped_char(character).into()),
- };
+ // Max of 6 digits.
+ // Minimum of 1 digit.
+ if unicode.len() > 6 || unicode.is_empty() {
+ return Err(ParserError::lexer_invalid_escaped_unicode_length(unicode).into());
+ }
+
+ if let Ok(hex) = u32::from_str_radix(&unicode, 16) {
+ if let Some(character) = std::char::from_u32(hex) {
+ // scalar
+ Ok((len, Char::Scalar(character)))
+ } else if hex <= 0x10FFFF {
+ Ok((len, Char::NonScalar(hex)))
} else {
- return Err(ParserError::lexer_unclosed_escaped_char().into());
+ Err(ParserError::lexer_invalid_character_exceeded_max_value(unicode).into())
}
+ } else {
+ Err(ParserError::lexer_expected_valid_hex_char(unicode).into())
+ }
+ }
+
+ fn eat_hex_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ let mut hex = String::new();
+ // Account for the chars '\' and 'x'.
+ let mut len = 2;
+
+ // At least one hex character necessary.
+ if let Some(c) = input.next_if(|c| c != &'\'') {
+ len += 1;
+ hex.push(c);
+ } else if let Some(c) = input.next() {
+ return Err(ParserError::lexer_expected_valid_hex_char(c).into());
+ } else {
+ return Err(ParserError::lexer_empty_input_tendril().into());
}
- if hex {
- let string = input_tendril.to_string();
- let hex_string = &string[2..string.len()];
-
- if hex_string.len() != 2 {
- return Err(ParserError::lexer_escaped_hex_incorrect_length(hex_string).into());
- }
-
- if let Ok(ascii_number) = u8::from_str_radix(hex_string, 16) {
- // According to RFC, we allow only values less than 128.
- if ascii_number > 127 {
- return Err(ParserError::lexer_expected_valid_hex_char(ascii_number).into());
- }
-
- return Ok(Char::Scalar(ascii_number as char));
- }
+ // Second hex character optional.
+ if let Some(c) = input.next_if(|c| c != &'\'') {
+ len += 1;
+ hex.push(c);
}
- if unicode {
- let string = input_tendril.to_string();
- if string.find('{').is_none() {
- return Err(ParserError::lexer_unopened_escaped_unicode_char(string).into());
- } else if string.find('}').is_none() {
- return Err(ParserError::lexer_unclosed_escaped_unicode_char(string).into());
+ if let Ok(ascii_number) = u8::from_str_radix(&hex, 16) {
+ // According to RFC, we allow only values less than 128.
+ if ascii_number > 127 {
+ return Err(ParserError::lexer_expected_valid_hex_char(hex).into());
}
- let unicode_number = &string[3..string.len() - 1];
- let len = unicode_number.len();
- if !(1..=6).contains(&len) {
- return Err(ParserError::lexer_invalid_escaped_unicode_length(unicode_number).into());
- }
-
- if let Ok(hex) = u32::from_str_radix(unicode_number, 16) {
- if let Some(character) = std::char::from_u32(hex) {
- // scalar
- return Ok(Char::Scalar(character));
- } else if hex <= 0x10FFFF {
- return Ok(Char::NonScalar(hex));
- } else {
- return Err(ParserError::lexer_invalid_character_exceeded_max_value(unicode_number).into());
- }
- }
+ Ok((len, Char::Scalar(ascii_number as char)))
+ } else {
+ Err(ParserError::lexer_expected_valid_hex_char(hex).into())
}
+ }
- if input_tendril.to_string().chars().count() != 1 {
- // If char doesn't close.
- return Err(ParserError::lexer_char_not_closed(&input_tendril[0..]).into());
- } else if let Some(character) = input_tendril.to_string().chars().next() {
- // If its a simple char.
- return Ok(Char::Scalar(character));
+ fn eat_escaped_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ match input.next() {
+ None => Err(ParserError::lexer_empty_input_tendril().into()),
+ // Length of 2 to account the '\'.
+ Some('0') => Ok((2, Char::Scalar(0 as char))),
+ Some('t') => Ok((2, Char::Scalar(9 as char))),
+ Some('n') => Ok((2, Char::Scalar(10 as char))),
+ Some('r') => Ok((2, Char::Scalar(13 as char))),
+ Some('\"') => Ok((2, Char::Scalar(34 as char))),
+ Some('\'') => Ok((2, Char::Scalar(39 as char))),
+ Some('\\') => Ok((2, Char::Scalar(92 as char))),
+ Some('u') => Self::eat_unicode_char(input),
+ Some('x') => Self::eat_hex_char(input),
+ Some(c) => Err(ParserError::lexer_expected_valid_escaped_char(c).into()),
}
+ }
- Err(ParserError::lexer_invalid_char(input_tendril.to_string()).into())
+ ///
+ /// Returns a `char` if a character can be eaten, otherwise returns [`None`].
+ ///
+ fn eat_char(input: &mut Peekable>) -> Result<(usize, Char)> {
+ match input.next() {
+ None => Err(ParserError::lexer_empty_input_tendril().into()),
+ Some('\\') => Self::eat_escaped_char(input),
+ Some(c) => Ok((c.len_utf8(), Char::Scalar(c))),
+ }
}
///
@@ -186,16 +203,21 @@ impl Token {
return Ok((1, Token::WhiteSpace));
}
Some('"') => {
- let mut string = Vec::new();
+ let mut string: Vec = Vec::new();
input.next();
- while let Some(c) = input.next_if(|c| c != &'"') {
- let character = leo_ast::Char::Scalar(c);
- string.push(character);
+ let mut len = 0;
+ while let Some(c) = input.peek() {
+ if c == &'"' {
+ break;
+ }
+ let (char_len, character) = Self::eat_char(&mut input)?;
+ len += char_len;
+ string.push(character.into());
}
if input.next_if_eq(&'"').is_some() {
- return Ok((string.len() + 2, Token::StringLit(string)));
+ return Ok((len + 2, Token::StringLit(string)));
}
return Err(ParserError::lexer_string_not_closed(string).into());
@@ -203,19 +225,16 @@ impl Token {
Some('\'') => {
input.next();
- if let Some(c) = input.next() {
- dbg!(&c);
- if input.next_if_eq(&'\'').is_some() {
- input.next();
- return Ok((c.len_utf8() + 2, Token::CharLit(Char::Scalar(c))));
- } else if let Some(c) = input.next() {
- return Err(ParserError::lexer_string_not_closed(c).into());
- } else {
- return Err(ParserError::lexer_empty_input_tendril().into());
- }
- }
+ let (len, character) = Self::eat_char(&mut input)?;
- return Err(ParserError::lexer_empty_input_tendril().into());
+ if input.next_if_eq(&'\'').is_some() {
+ input.next();
+ return Ok((len + 2, Token::CharLit(character)));
+ } else if let Some(c) = input.next() {
+ return Err(ParserError::lexer_string_not_closed(c).into());
+ } else {
+ return Err(ParserError::lexer_empty_input_tendril().into());
+ }
}
Some(x) if x.is_ascii_digit() => {
return Self::eat_integer(&mut input);
diff --git a/compiler/parser/src/tokenizer/mod.rs b/compiler/parser/src/tokenizer/mod.rs
index 919c883fd6..e3daac5682 100644
--- a/compiler/parser/src/tokenizer/mod.rs
+++ b/compiler/parser/src/tokenizer/mod.rs
@@ -133,10 +133,6 @@ mod tests {
r#"
'a'
'😭'
- '\u{10001F}'
- '\x7f'
- '\x00'
- '\x37'
"test"
"test{}test"
"test{}"
@@ -235,13 +231,12 @@ mod tests {
assert_eq!(
output,
- r#"'a' '😭' '\u{10001F}' "test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut & return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
+ r#"'a' '😭' "test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut & return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || ? // test
/* test */ // "#
);
});
}
- #[test]
fn test_spans() {
create_session_if_not_set_then(|_| {
let raw = r#"
diff --git a/compiler/parser/src/tokenizer/token.rs b/compiler/parser/src/tokenizer/token.rs
index a9c3e17c86..167fd04dff 100644
--- a/compiler/parser/src/tokenizer/token.rs
+++ b/compiler/parser/src/tokenizer/token.rs
@@ -39,7 +39,7 @@ impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Scalar(c) => write!(f, "{}", c),
- Self::NonScalar(c) => write!(f, "{}", c),
+ Self::NonScalar(c) => write!(f, "{:X}", c),
}
}
}
From 0e7d36a61c97fe1daa8c472512a1680600792013 Mon Sep 17 00:00:00 2001
From: gluax <16431709+gluax@users.noreply.github.com>
Date: Wed, 16 Mar 2022 13:02:56 -0700
Subject: [PATCH 04/13] current tests pass
---
compiler/parser/src/tokenizer/lexer.rs | 21 ++--
compiler/parser/src/tokenizer/mod.rs | 1 +
.../parser/expression/literal/char.leo.out | 102 +++++++++---------
.../expression/literal/char_fail.leo.out | 62 +++++------
.../expression/literal/char_parse.leo.out | 102 +++++++++---------
.../parser/expression/literal/string.leo.out | 4 +-
.../expression/literal/string_fail.leo.out | 16 +--
.../expression/literal/string_parse.leo.out | 6 +-
.../parser/parser/program/pipe_eof.leo.out | 2 +-
.../parser/parser/program/q_eof.leo.out | 2 +-
.../parser/parser/program/sq_eof.leo.out | 2 +-
.../program/unclosed_unicode_eof_fail.leo.out | 2 +-
.../parser/serialize/palindrome.leo.out | 10 ++
13 files changed, 175 insertions(+), 157 deletions(-)
diff --git a/compiler/parser/src/tokenizer/lexer.rs b/compiler/parser/src/tokenizer/lexer.rs
index db762f4663..4750d86105 100644
--- a/compiler/parser/src/tokenizer/lexer.rs
+++ b/compiler/parser/src/tokenizer/lexer.rs
@@ -90,7 +90,7 @@ impl Token {
// Account for the chars '\' and 'x'.
let mut len = 2;
- // At least one hex character necessary.
+ // First hex character.
if let Some(c) = input.next_if(|c| c != &'\'') {
len += 1;
hex.push(c);
@@ -100,10 +100,14 @@ impl Token {
return Err(ParserError::lexer_empty_input_tendril().into());
}
- // Second hex character optional.
+ // Second hex character.
if let Some(c) = input.next_if(|c| c != &'\'') {
len += 1;
hex.push(c);
+ } else if let Some(c) = input.next() {
+ return Err(ParserError::lexer_expected_valid_hex_char(c).into());
+ } else {
+ return Err(ParserError::lexer_empty_input_tendril().into());
}
if let Ok(ascii_number) = u8::from_str_radix(&hex, 16) {
@@ -313,10 +317,9 @@ impl Token {
return Ok((1, Token::Dot));
}
Some(c) if c == &'/' => {
- let mut comment = String::from(*c);
input.next();
- if let Some(c) = input.next_if_eq(&'/') {
- comment.push(c);
+ if input.next_if_eq(&'/').is_some() {
+ let mut comment = String::from("//");
while let Some(c) = input.next_if(|c| c != &'\n') {
comment.push(c);
@@ -324,12 +327,12 @@ impl Token {
if let Some(newline) = input.next_if_eq(&'\n') {
comment.push(newline);
- return Ok((comment.len() + 1, Token::CommentLine(comment)));
+ return Ok((comment.len(), Token::CommentLine(comment)));
}
return Ok((comment.len(), Token::CommentLine(comment)));
- } else if let Some(c) = input.next_if_eq(&'*') {
- comment.push(c);
+ } else if input.next_if_eq(&'*').is_some() {
+ let mut comment = String::from("/*");
if input.peek().is_none() {
return Err(ParserError::lexer_empty_block_comment().into());
@@ -348,7 +351,7 @@ impl Token {
if !ended {
return Err(ParserError::lexer_block_comment_does_not_close_before_eof(comment).into());
}
- return Ok((comment.len() + 4, Token::CommentBlock(comment)));
+ return Ok((comment.len(), Token::CommentBlock(comment)));
} else if input.next_if_eq(&'=').is_some() {
return Ok((2, Token::DivEq));
}
diff --git a/compiler/parser/src/tokenizer/mod.rs b/compiler/parser/src/tokenizer/mod.rs
index e3daac5682..bd935311aa 100644
--- a/compiler/parser/src/tokenizer/mod.rs
+++ b/compiler/parser/src/tokenizer/mod.rs
@@ -237,6 +237,7 @@ mod tests {
});
}
+ #[test]
fn test_spans() {
create_session_if_not_set_then(|_| {
let raw = r#"
diff --git a/tests/expectations/parser/parser/expression/literal/char.leo.out b/tests/expectations/parser/parser/expression/literal/char.leo.out
index fa87952664..3d84a2cece 100644
--- a/tests/expectations/parser/parser/expression/literal/char.leo.out
+++ b/tests/expectations/parser/parser/expression/literal/char.leo.out
@@ -2,54 +2,54 @@
namespace: Token
expectation: Pass
outputs:
- - "'a' @ 1:1-4"
- - "'Z' @ 1:1-4"
- - "'\"' @ 1:1-5"
- - "'' @ 1:1-5"
- - "'' @ 1:1-5"
- - "'\u0000' @ 1:1-5"
- - "'\u000f' @ 1:1-8"
- - "'' @ 1:1-6"
- - "'å' @ 1:1-9"
- - "'å' @ 1:1-5"
- - "'Ӡ' @ 1:1-10"
- - "'Ӡ' @ 1:1-5"
- - "'❤' @ 1:1-11"
- - "'❤' @ 1:1-6"
- - "'😢' @ 1:1-12"
- - "'😭' @ 1:1-7"
- - "'' @ 1:1-13"
- - "'*' @ 1:1-7"
- - "'\u007f' @ 1:1-7"
- - "'\u0000' @ 1:1-7"
- - "'\u0001' @ 1:1-7"
- - "'\u0002' @ 1:1-7"
- - "'\u0003' @ 1:1-7"
- - "'\u0004' @ 1:1-7"
- - "'\u0005' @ 1:1-7"
- - "'\u0006' @ 1:1-7"
- - "'\u0007' @ 1:1-7"
- - "'\u0010' @ 1:1-7"
- - "'\u0011' @ 1:1-7"
- - "'\u0012' @ 1:1-7"
- - "'\u0013' @ 1:1-7"
- - "'\u0014' @ 1:1-7"
- - "'\u0015' @ 1:1-7"
- - "'\u0016' @ 1:1-7"
- - "'\u0017' @ 1:1-7"
- - "'' @ 1:1-7"
- - "'!' @ 1:1-7"
- - "'\"' @ 1:1-7"
- - "'#' @ 1:1-7"
- - "'$' @ 1:1-7"
- - "'%' @ 1:1-7"
- - "'&' @ 1:1-7"
- - "''' @ 1:1-7"
- - "'0' @ 1:1-7"
- - "'1' @ 1:1-7"
- - "'2' @ 1:1-7"
- - "'3' @ 1:1-7"
- - "'4' @ 1:1-7"
- - "'5' @ 1:1-7"
- - "'6' @ 1:1-7"
- - "'7' @ 1:1-7"
+ - "''a'' @ 1:1-4"
+ - "''Z'' @ 1:1-4"
+ - "''\"'' @ 1:1-5"
+ - "''\t'' @ 1:1-5"
+ - "''\r'' @ 1:1-5"
+ - "''\u0000'' @ 1:1-5"
+ - "''\u000f'' @ 1:1-8"
+ - "'''' @ 1:1-6"
+ - "''å'' @ 1:1-9"
+ - "''å'' @ 1:1-5"
+ - "''Ӡ'' @ 1:1-10"
+ - "''Ӡ'' @ 1:1-5"
+ - "''❤'' @ 1:1-11"
+ - "''❤'' @ 1:1-6"
+ - "''😢'' @ 1:1-12"
+ - "''😭'' @ 1:1-7"
+ - "'''' @ 1:1-13"
+ - "''*'' @ 1:1-7"
+ - "''\u007f'' @ 1:1-7"
+ - "''\u0000'' @ 1:1-7"
+ - "''\u0001'' @ 1:1-7"
+ - "''\u0002'' @ 1:1-7"
+ - "''\u0003'' @ 1:1-7"
+ - "''\u0004'' @ 1:1-7"
+ - "''\u0005'' @ 1:1-7"
+ - "''\u0006'' @ 1:1-7"
+ - "''\u0007'' @ 1:1-7"
+ - "''\u0010'' @ 1:1-7"
+ - "''\u0011'' @ 1:1-7"
+ - "''\u0012'' @ 1:1-7"
+ - "''\u0013'' @ 1:1-7"
+ - "''\u0014'' @ 1:1-7"
+ - "''\u0015'' @ 1:1-7"
+ - "''\u0016'' @ 1:1-7"
+ - "''\u0017'' @ 1:1-7"
+ - "'' '' @ 1:1-7"
+ - "''!'' @ 1:1-7"
+ - "''\"'' @ 1:1-7"
+ - "''#'' @ 1:1-7"
+ - "''$'' @ 1:1-7"
+ - "''%'' @ 1:1-7"
+ - "''&'' @ 1:1-7"
+ - "''''' @ 1:1-7"
+ - "''0'' @ 1:1-7"
+ - "''1'' @ 1:1-7"
+ - "''2'' @ 1:1-7"
+ - "''3'' @ 1:1-7"
+ - "''4'' @ 1:1-7"
+ - "''5'' @ 1:1-7"
+ - "''6'' @ 1:1-7"
+ - "''7'' @ 1:1-7"
diff --git a/tests/expectations/parser/parser/expression/literal/char_fail.leo.out b/tests/expectations/parser/parser/expression/literal/char_fail.leo.out
index c7aaedc6ce..61b1087364 100644
--- a/tests/expectations/parser/parser/expression/literal/char_fail.leo.out
+++ b/tests/expectations/parser/parser/expression/literal/char_fail.leo.out
@@ -2,22 +2,22 @@
namespace: Token
expectation: Fail
outputs:
- - "Error [EPAR0370028]: Expected a closed char but found `'\\'`."
- - "Error [EPAR0370028]: Expected a closed char but found `'a`."
- "Error [EPAR0370024]: Expected more characters to lex but found none."
- - "Error [EPAR0370036]: Expected a valid hex character but found `154`."
- - "Error [EPAR0370035]: Could not lex the following escaped hex due to being given more than two chars: `7`."
- - "Error [EPAR0370028]: Expected a closed char but found `\\x7g`."
- - "Error [EPAR0370035]: Could not lex the following escaped hex due to being given more than two chars: `z`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `128`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `193`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `194`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `223`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `192`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `224`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `159`."
- - "Error [EPAR0370028]: Expected a closed char but found `abcdefg`."
- - "Error [EPAR0370033]: Could not lex the following escaped char due to being given more than one char: `t\\t`."
+ - "Error [EPAR0370024]: Expected more characters to lex but found none."
+ - "Error [EPAR0370024]: Expected more characters to lex but found none."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `9A`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `'`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `7g`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `'`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `80`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `c1`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `c2`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `DF`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `C0`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `e0`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `9f`."
+ - "Error [EPAR0370027]: Expected a closed string but found `'b'`."
+ - "Error [EPAR0370027]: Expected a closed string but found `'\\\\'`."
- "Error [EPAR0370026]: Expected a valid escape character but found `a`."
- "Error [EPAR0370026]: Expected a valid escape character but found `z`."
- "Error [EPAR0370026]: Expected a valid escape character but found `A`."
@@ -25,20 +25,20 @@ outputs:
- "Error [EPAR0370026]: Expected a valid escape character but found `1`."
- "Error [EPAR0370026]: Expected a valid escape character but found `9`."
- "Error [EPAR0370026]: Expected a valid escape character but found `*`."
- - "Error [EPAR0370035]: Could not lex the following escaped hex due to being given more than two chars: ``."
- - "Error [EPAR0370026]: Expected a valid escape character but found `39`."
- - "Error [EPAR0370038]: The escaped unicode char `bbbbb}\\u{aaaa` is not within valid length of [1, 6]."
- - "Error [EPAR0370026]: Expected a valid escape character but found `122`."
- - "Error [EPAR0370026]: Expected a valid escape character but found `49`."
- - "Error [EPAR0370026]: Expected a valid escape character but found `49`."
- - "Error [EPAR0370037]: There was no closing `}` after a escaped unicode `\\u{2764z`."
- - "Error [EPAR0370028]: Expected a closed char but found `\\u{276g}`."
- - "Error [EPAR0370026]: Expected a valid escape character but found `48`."
- - "Error [EPAR0370026]: Expected a valid escape character but found `48`."
- - "Error [EPAR0370026]: Expected a valid escape character but found `57`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `'`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `'`."
+ - "Error [EPAR0370027]: Expected a closed string but found `'\\\\'`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `z`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `1`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `1`."
+ - "Error [EPAR0370037]: There was no closing `}` after a escaped unicode `2764z'`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `276g`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `0`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `0`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `9`."
- "Error [EPAR0370039]: The escaped unicode char `110000` is greater than 0x10FFFF."
- - "Error [EPAR0370026]: Expected a valid escape character but found `125`."
- - "Error [EPAR0370037]: There was no closing `}` after a escaped unicode `\\u{af🦀`."
- - "Error [EPAR0370029]: Expected valid character but found `'🦀\\n`."
- - "Error [EPAR0370029]: Expected valid character but found `'🦀1🦀`."
- - "Error [EPAR0370029]: Expected valid character but found `'😭😂`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `}`."
+ - "Error [EPAR0370037]: There was no closing `}` after a escaped unicode `af🦀'`."
+ - "Error [EPAR0370027]: Expected a closed string but found `'\\\\'`."
+ - "Error [EPAR0370027]: Expected a closed string but found `'1'`."
+ - "Error [EPAR0370027]: Expected a closed string but found `'😂'`."
diff --git a/tests/expectations/parser/parser/expression/literal/char_parse.leo.out b/tests/expectations/parser/parser/expression/literal/char_parse.leo.out
index fa87952664..3d84a2cece 100644
--- a/tests/expectations/parser/parser/expression/literal/char_parse.leo.out
+++ b/tests/expectations/parser/parser/expression/literal/char_parse.leo.out
@@ -2,54 +2,54 @@
namespace: Token
expectation: Pass
outputs:
- - "'a' @ 1:1-4"
- - "'Z' @ 1:1-4"
- - "'\"' @ 1:1-5"
- - "'' @ 1:1-5"
- - "'' @ 1:1-5"
- - "'\u0000' @ 1:1-5"
- - "'\u000f' @ 1:1-8"
- - "'' @ 1:1-6"
- - "'å' @ 1:1-9"
- - "'å' @ 1:1-5"
- - "'Ӡ' @ 1:1-10"
- - "'Ӡ' @ 1:1-5"
- - "'❤' @ 1:1-11"
- - "'❤' @ 1:1-6"
- - "'😢' @ 1:1-12"
- - "'😭' @ 1:1-7"
- - "'' @ 1:1-13"
- - "'*' @ 1:1-7"
- - "'\u007f' @ 1:1-7"
- - "'\u0000' @ 1:1-7"
- - "'\u0001' @ 1:1-7"
- - "'\u0002' @ 1:1-7"
- - "'\u0003' @ 1:1-7"
- - "'\u0004' @ 1:1-7"
- - "'\u0005' @ 1:1-7"
- - "'\u0006' @ 1:1-7"
- - "'\u0007' @ 1:1-7"
- - "'\u0010' @ 1:1-7"
- - "'\u0011' @ 1:1-7"
- - "'\u0012' @ 1:1-7"
- - "'\u0013' @ 1:1-7"
- - "'\u0014' @ 1:1-7"
- - "'\u0015' @ 1:1-7"
- - "'\u0016' @ 1:1-7"
- - "'\u0017' @ 1:1-7"
- - "'' @ 1:1-7"
- - "'!' @ 1:1-7"
- - "'\"' @ 1:1-7"
- - "'#' @ 1:1-7"
- - "'$' @ 1:1-7"
- - "'%' @ 1:1-7"
- - "'&' @ 1:1-7"
- - "''' @ 1:1-7"
- - "'0' @ 1:1-7"
- - "'1' @ 1:1-7"
- - "'2' @ 1:1-7"
- - "'3' @ 1:1-7"
- - "'4' @ 1:1-7"
- - "'5' @ 1:1-7"
- - "'6' @ 1:1-7"
- - "'7' @ 1:1-7"
+ - "''a'' @ 1:1-4"
+ - "''Z'' @ 1:1-4"
+ - "''\"'' @ 1:1-5"
+ - "''\t'' @ 1:1-5"
+ - "''\r'' @ 1:1-5"
+ - "''\u0000'' @ 1:1-5"
+ - "''\u000f'' @ 1:1-8"
+ - "'''' @ 1:1-6"
+ - "''å'' @ 1:1-9"
+ - "''å'' @ 1:1-5"
+ - "''Ӡ'' @ 1:1-10"
+ - "''Ӡ'' @ 1:1-5"
+ - "''❤'' @ 1:1-11"
+ - "''❤'' @ 1:1-6"
+ - "''😢'' @ 1:1-12"
+ - "''😭'' @ 1:1-7"
+ - "'''' @ 1:1-13"
+ - "''*'' @ 1:1-7"
+ - "''\u007f'' @ 1:1-7"
+ - "''\u0000'' @ 1:1-7"
+ - "''\u0001'' @ 1:1-7"
+ - "''\u0002'' @ 1:1-7"
+ - "''\u0003'' @ 1:1-7"
+ - "''\u0004'' @ 1:1-7"
+ - "''\u0005'' @ 1:1-7"
+ - "''\u0006'' @ 1:1-7"
+ - "''\u0007'' @ 1:1-7"
+ - "''\u0010'' @ 1:1-7"
+ - "''\u0011'' @ 1:1-7"
+ - "''\u0012'' @ 1:1-7"
+ - "''\u0013'' @ 1:1-7"
+ - "''\u0014'' @ 1:1-7"
+ - "''\u0015'' @ 1:1-7"
+ - "''\u0016'' @ 1:1-7"
+ - "''\u0017'' @ 1:1-7"
+ - "'' '' @ 1:1-7"
+ - "''!'' @ 1:1-7"
+ - "''\"'' @ 1:1-7"
+ - "''#'' @ 1:1-7"
+ - "''$'' @ 1:1-7"
+ - "''%'' @ 1:1-7"
+ - "''&'' @ 1:1-7"
+ - "''''' @ 1:1-7"
+ - "''0'' @ 1:1-7"
+ - "''1'' @ 1:1-7"
+ - "''2'' @ 1:1-7"
+ - "''3'' @ 1:1-7"
+ - "''4'' @ 1:1-7"
+ - "''5'' @ 1:1-7"
+ - "''6'' @ 1:1-7"
+ - "''7'' @ 1:1-7"
diff --git a/tests/expectations/parser/parser/expression/literal/string.leo.out b/tests/expectations/parser/parser/expression/literal/string.leo.out
index a346d6b703..c6b41bc0d5 100644
--- a/tests/expectations/parser/parser/expression/literal/string.leo.out
+++ b/tests/expectations/parser/parser/expression/literal/string.leo.out
@@ -15,5 +15,5 @@ outputs:
- "'\"\n\"' @ 1:1-7"
- "'\"\u007f\"' @ 1:1-7"
- "'\"aa \\ \" \n aa \t \r \u0000\"' @ 1:1-28"
- - "'\"test \"' @ 1:1-15"
- - "'\"\"' @ 1:1-15"
+ - "'\"test 😒€\"' @ 1:1-15"
+ - "'\"😭😂😘\"' @ 1:1-15"
diff --git a/tests/expectations/parser/parser/expression/literal/string_fail.leo.out b/tests/expectations/parser/parser/expression/literal/string_fail.leo.out
index 8865fffa53..6a71ca45fb 100644
--- a/tests/expectations/parser/parser/expression/literal/string_fail.leo.out
+++ b/tests/expectations/parser/parser/expression/literal/string_fail.leo.out
@@ -2,12 +2,12 @@
namespace: Token
expectation: Fail
outputs:
- - "Error [EPAR0370027]: Expected a closed string but found `\"Hello world!`."
- - "Error [EPAR0370027]: Expected a closed string but found `\"\\\"`."
+ - "Error [EPAR0370027]: Expected a closed string but found `[Scalar('H'), Scalar('e'), Scalar('l'), Scalar('l'), Scalar('o'), Scalar(' '), Scalar('w'), Scalar('o'), Scalar('r'), Scalar('l'), Scalar('d'), Scalar('!')]`."
+ - "Error [EPAR0370027]: Expected a closed string but found `[Scalar('\"')]`."
- "Error [EPAR0370026]: Expected a valid escape character but found `l`."
- - "Error [EPAR0370027]: Expected a closed string but found `\"\\uaaa\"`."
- - "Error [EPAR0370027]: Expected a closed string but found `\"\\u\"`."
- - "Error [EPAR0370036]: Expected a valid hex character but found `255`."
- - "Error [EPAR0370027]: Expected a closed string but found `\"\\x\"`."
- - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `\\u}`."
- - "Error [EPAR0370043]: There was an emoji found in the escaped unicode character: `\"\\u6🦀`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `a`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `\"`."
+ - "Error [EPAR0370036]: Expected a valid hex character but found `FF`."
+ - "Error [EPAR0370024]: Expected more characters to lex but found none."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `}`."
+ - "Error [EPAR0370042]: There was no opening `{` after starting an escaped unicode `6`."
diff --git a/tests/expectations/parser/parser/expression/literal/string_parse.leo.out b/tests/expectations/parser/parser/expression/literal/string_parse.leo.out
index 2f4bf81ada..cd98aa67b8 100644
--- a/tests/expectations/parser/parser/expression/literal/string_parse.leo.out
+++ b/tests/expectations/parser/parser/expression/literal/string_parse.leo.out
@@ -177,6 +177,8 @@ outputs:
- Scalar: 115
- Scalar: 116
- Scalar: 32
+ - Scalar: 128530
+ - Scalar: 8364
- span:
line_start: 1
line_stop: 1
@@ -186,7 +188,9 @@ outputs:
content: "\"test 😒€\""
- Value:
String:
- - []
+ - - Scalar: 128557
+ - Scalar: 128514
+ - Scalar: 128536
- span:
line_start: 1
line_stop: 1
diff --git a/tests/expectations/parser/parser/program/pipe_eof.leo.out b/tests/expectations/parser/parser/program/pipe_eof.leo.out
index 2103e7b671..fce74aefd6 100644
--- a/tests/expectations/parser/parser/program/pipe_eof.leo.out
+++ b/tests/expectations/parser/parser/program/pipe_eof.leo.out
@@ -2,4 +2,4 @@
namespace: Parse
expectation: Fail
outputs:
- - "Error [EPAR0370032]: Could not lex the following content: `|`."
+ - "Error [EPAR0370024]: Expected more characters to lex but found none."
diff --git a/tests/expectations/parser/parser/program/q_eof.leo.out b/tests/expectations/parser/parser/program/q_eof.leo.out
index 68f7c3b559..253e235d75 100644
--- a/tests/expectations/parser/parser/program/q_eof.leo.out
+++ b/tests/expectations/parser/parser/program/q_eof.leo.out
@@ -2,4 +2,4 @@
namespace: Parse
expectation: Fail
outputs:
- - "Error [EPAR0370027]: Expected a closed string but found `\"`."
+ - "Error [EPAR0370027]: Expected a closed string but found `[]`."
diff --git a/tests/expectations/parser/parser/program/sq_eof.leo.out b/tests/expectations/parser/parser/program/sq_eof.leo.out
index 1bab5ffabb..fce74aefd6 100644
--- a/tests/expectations/parser/parser/program/sq_eof.leo.out
+++ b/tests/expectations/parser/parser/program/sq_eof.leo.out
@@ -2,4 +2,4 @@
namespace: Parse
expectation: Fail
outputs:
- - "Error [EPAR0370028]: Expected a closed char but found `'`."
+ - "Error [EPAR0370024]: Expected more characters to lex but found none."
diff --git a/tests/expectations/parser/parser/program/unclosed_unicode_eof_fail.leo.out b/tests/expectations/parser/parser/program/unclosed_unicode_eof_fail.leo.out
index ad932d117a..fce74aefd6 100644
--- a/tests/expectations/parser/parser/program/unclosed_unicode_eof_fail.leo.out
+++ b/tests/expectations/parser/parser/program/unclosed_unicode_eof_fail.leo.out
@@ -2,4 +2,4 @@
namespace: Parse
expectation: Fail
outputs:
- - "Error [EPAR0370026]: Expected a valid escape character but found `117`."
+ - "Error [EPAR0370024]: Expected more characters to lex but found none."
diff --git a/tests/expectations/parser/parser/serialize/palindrome.leo.out b/tests/expectations/parser/parser/serialize/palindrome.leo.out
index df0ea483e8..6ecc69d502 100644
--- a/tests/expectations/parser/parser/serialize/palindrome.leo.out
+++ b/tests/expectations/parser/parser/serialize/palindrome.leo.out
@@ -416,6 +416,10 @@ outputs:
arguments:
- Value:
String:
+ - Scalar: 128512
+ - Scalar: 128512
+ - Scalar: 128512
+ - Scalar: 128512
- Scalar: 128512
- Scalar: 32
- Scalar: 32
@@ -426,6 +430,12 @@ outputs:
- Scalar: 32
- Scalar: 32
- Scalar: 32
+ - Scalar: 32
+ - Scalar: 128512
+ - Scalar: 128512
+ - Scalar: 128512
+ - Scalar: 128512
+ - Scalar: 128512
- Console:
function:
Assert:
From 34e1b39805d217f19c4d149904e0c3d0e90ff965 Mon Sep 17 00:00:00 2001
From: gluax <16431709+gluax@users.noreply.github.com>
Date: Wed, 16 Mar 2022 15:04:49 -0700
Subject: [PATCH 05/13] some cleanup
---
compiler/parser/default.profraw | Bin 0 -> 3142104 bytes
compiler/parser/src/tokenizer/lexer.rs | 23 +----
covered.yml | Bin 0 -> 46242 bytes
default.profraw | Bin 0 -> 10854168 bytes
leo/errors/covered.yml | Bin 0 -> 46060 bytes
leo/errors/src/parser/parser_errors.rs | 51 +---------
.../parser/circuits/mixed_order_fail.leo.out | 2 +-
.../parser/circuits/mut_self_fail.leo.out | 2 +-
.../circuits/self_not_first_fail.leo.out | 2 +-
.../parser/expression/array_init_fail.leo.out | 4 +-
.../expression/literal/char_fail.leo.out | 88 +++++++++---------
.../expression/literal/comment_fail.leo.out | 18 ++--
.../expression/literal/int_fail.leo.out | 6 +-
.../expression/literal/string_fail.leo.out | 24 ++---
.../functions/annotated_arg_not_ident.leo.out | 2 +-
.../functions/annotated_context_fail.leo.out | 2 +-
.../parser/functions/escape_fail.leo.out | 2 +-
.../parser/program/backslash_eof.leo.out | 2 +-
.../parser/parser/program/dollar_eof.leo.out | 2 +-
.../parser/program/escape_u8_eof.leo.out | 2 +-
.../parser/parser/program/hex_eof.leo.out | 2 +-
.../parser/parser/program/pipe_eof.leo.out | 2 +-
.../parser/parser/program/q_eof.leo.out | 2 +-
.../parser/parser/program/sq_eof.leo.out | 2 +-
.../parser/parser/program/tilde_eof.leo.out | 2 +-
.../program/unclosed_unicode_eof_fail.leo.out | 2 +-
.../parser/statement/assign_fail.leo.out | 2 +-
.../parser/statement/definition_fail.leo.out | 48 +++++-----
.../parser/statement/expression_fail.leo.out | 4 +-
.../parser/statement/hex_int_fail.leo.out | 6 +-
.../parser/statement/let_mut_recover.leo.out | 2 +-
tests/test-framework/src/bin/errcov.rs | 2 +-
32 files changed, 122 insertions(+), 186 deletions(-)
create mode 100644 compiler/parser/default.profraw
create mode 100644 covered.yml
create mode 100644 default.profraw
create mode 100644 leo/errors/covered.yml
diff --git a/compiler/parser/default.profraw b/compiler/parser/default.profraw
new file mode 100644
index 0000000000000000000000000000000000000000..d9ada89c28c30132003615ede889b3ac16f0bcd6
GIT binary patch
literal 3142104
zcma&vc{o+;`#*3q86rc-ka;X*+-8|F&$LCx5=n%#Ng;~Nnc9XjW{i-sQz4lvL!wPc
zlzE=oWcYo)-`{!8b$zd^<@ua{T&(&;d%Jpn~c$qoo>gjffNAX2G*4~^M
z1il#Ji3r>2lj%kGfscatNsB-p^Y*fB;N2nK7dU
zzr-(w+oH#aPRqRhg}zVLiOL*7kIpjpO%dXwIuM<5dVkG
z*@dyKGzoY-#J{_h!1RRIN)GsTi038K%X|0a6BY0{h@aauIGsqAM+
zSpYm1;(t{ze2w|O;sbmX#P`bd$mX$qXIqRpToR|axP(FQ6#uLjoyGW-)J$XfLG5yu
z1s%yMBv1N}*EB9BiHn~}A(c4=cpr!#E{Utp^wORJ-VEZ^
zhW9_y7#;QnUJBy1T>=Bk-8$TXXMlLsAOQLj0u+wGUo1V{yQTLVVl>HO?O|(R{%BK)l5j&W!Jv=3U^OAl{0M
z@-v!GZytCvi2wLu$oPFU0EpoWsQ
z^Xtakn#W!f#ilSIcyPXp#g+a(;pTG-_~mDRpJ(8^?p5r)?|M20d=JDs&l^uDr0%~4
zz7paoqVK=G5>a>;_(X_bOiAprh%*ob-Us3v$WGGz{LHlpycxtN%YSMM;@wgJUJBxQ
zFQ1Lh5+7#+o&n+q2r2mY-kU%XTbYFd^;JlM8nd(qfJ>#KB?IZ$z}*1ynZn}Dc9I3X9@%!
z1@SYNnLd0!e77BV9*DO;6IDqZcfJgGDu{OwT04GgH8>0S}sgIwC*S1
z@esc=*up|R?NSJQBgFstky<~iIsR||#So9LzI5cmBbSGuej>yVS7m7Ld(>_MkAe74
z3*jUti9*W2J3&15dsJ@a^v-wS^&tL9P=@OHn`dW%mxB1k*Qd}qO19>}vqAhJtKvuZ
zw`R#>+Fxm`MA7;XA$YJKt{VpqQf$rN1Agr3-{%4Nprfn~JKyMsfp3KPQ(Kcx_sAY)
z1D^x&ML{RUrJ3w&fDeWEz=em$Bi1tpfp>y<|HHeweqVNHfY*Td#0h)*Y459tfaihu
z5Z6PuFVa5n1fB@uwQ9=>pX9}?0Y6rDaQ@Ey+oCzVt(w3$LVU?_Puo8)x66Ugf%uj8
zU7B~?b}j)Q3h@b7uiR0kEJp$F1o4WZCnrlqBxSZ^YsgPZo&6;Ix=Hy7
z@FEa@<*9xxvmpN}@C*>&+UB!V+O5q3JfZaO`GbC}KRyml6HEO7{20Vz9#9owhjCAV
zZ-@Bu{^T#qT$5hFS3>-m>ccfpe$}-DkA-+vVILX-tx_KFp%8EDfg!Wg*{K2E9pVil
znpc$4eJy}DgZRN5whi@)6K24pAbxA=gDCm_sw40`5Fgp_kk)}&q8@lEi0{8NqnRo3
z^O}p!v#+cwb8^ci2hYQ~5Bd%gaYl{6<00OIa?t20!zFLv8zElVw~$iqS!yZp#Snjw
zNGCRjvcnnpM2LU=^+%^Q(F!Z@7>NJ=vvT+tqo4rrP7rT7H!N^6_e}}#dJuoobWm_@
zsXrEYDTp6B6^vhD7gYeB4dOE*6EE*LXAJ{S1o87NV-32(DSNZI!qh_J__RZF6boq(%(w}-Us6K
zmMO(*esFyQ9u4v2g(}LAh5Xon*MRuIM%;?{KNuP^8)q``EgZ(@N9_)3W9=x|oq
zyvhMC4~U{5-ip|BN+Yu+8Ps=zcr^AWuhKRDBH&RF-@Ir?{kF^b-%p(wAYN8pFXDmx
zt6ETh`pLoh(B=+w3C3rw1K$Yo!>lqF)#=$MfyY98+w%x{ZF##);C&$eb)K>C{!(@-
z@OltmnGsMyY1YR9JP*X*V40znC{^`2l*
z@khvw3LoIBZGhK+c;|<9*6m3Q{=lay7XW*Cf4$de0bMx)d2k&x#Z-;m;
zD(&}dREAE!GxWxb157^UAb7B$UFWWR@KfPjgJ)7Io8&|CuqXr%KL0{94Sq_Qn|-C&
zd{G-+-YJ0O!ROKM;#ZV?%{OG~FWybfJ@4ckiclZ)BkS+t<%&+fe!}mV$>Tx~R(Avs
z?pLe?X-^q-)f%XV_Ty1`WU_h)9(>+W@J+0;Rz+O%3$M=NJ}u={6oLmY5W0db3Po0^
z431ysi8l6kg^<<@^b_>p)w+_{bd!7^SLO|B6OJB)`rvpSRj)JDtTxu@vtCrQ9JmmS
zL+~I^hPL75V%j$zn-?#|EWUdXQVH=>71b=$Qc{mDJ$6hda;K0ehIqR17^=%35+j4=
z<@jDJ43{F!kJvv0*-5D#Rz0*@U8nT@&7#w#m_~&82lM0B)IP!O+k3iJtb)U`AYcZ`
zgM9j5Nz(}*nc7pj=a!~e{!XbFp+4{?{XI0^gJ
z*4#RN=uY@K$52Rp&CI2w&bplS+{&BZm~L|QAnhlhpJ!ODwsvCAAJk@o%_f`rhGq!$
zK|h8RiLxc9j=3#vec>2vvOpp2KcN2E$fI}#{<*uVrG(FX68N&kaalUMqJ`CC+tHan6B{XEH`XW)AIiJWrIm?(e4K8*^Y
zK6t*xo@?dv5hH@!x`_kdKATt%Qhkt*+;nU=3FCY*Hmzf1kazYR144b!j|kb1y6%fd
zWZG`~CVTJ5Nf8j{AJh+ds5a|>*KX%%p6}^W;2@nw@Sy&erbG_^Gamw(WUG4d;zz@g
z&UaAXgOTzl<9MOO0%^q4fv#)Jjgb0>ZIdRRy$aqvXCCR&{m^L(>HG!#phvEC1=XE(
zl?ydk5>LaYCPL~{SKdtiwnu2eX_BDKObgK%1P}Tz-U=h`%XP#Qo0Q-Az5nc-JAw!O
zC{cxRwNr3cj!Hc}Z7VV~WrpBE{mqLQGg>eCRO=V(_s$w+&uSoe%76AB>)pLATbsRf
z?Y1lJr(=}g=^=Q~PiaSD@!90_coku<89w!M+foP~^n=?-*|$ol?4#xIBitKb@aIAB
z;QSK1VP@mYYOIMlVZ5}*q&LEb;6Z(Xe$L2mUUAw5*Ozl&k$VbKA$ZWg@zd6Y_mX6#
zk27d8_Nw^!mUI7p{Q&B#iL6RVNf4LBv{&xq?Ti5Bw9K(t35jL7%hYwb^T(3NDQZ9_&9#G2*-}
zqj_Jv8QY&9lQYI5c~IY;``5Vbf=rRLHSvf#PxwMHLVe(a@2iw{MD;$JpxaSACQf@Z
z5#l|fykxBE$4+k@8RTwt3;c~i@F4$7pFbJCWV@#NU}DfWn=AOWJH+cM8qLR^SxpH!
zR!$gd8+Jn@crb5~N+?^BH}`*Vb2CtVbDfW?8W7J&
zSYv)Qa?d_J$FOp&osvlk;!9>{kE1`Pa*)2>W>;dJ59dMfp#MuVaeGE5C}%znW43qg
ztpwO0elj!k=B9PzpN#bGduhBkOBoP6Sl?zz$)jU8J`Bv44c(5tWe`fp`TO|-_LGG*
z$s+TDpE6~R89Jg#$47}E^-FGCT8>iqRjVnd8>kxQ=s1l~ALO68vLvHB6vdM3Y-#uE
zlJpBa#A}D`w}{XDZuvxM?X3`c>1#WJ2gf_YIw-cnqx0aSrZvDSr#A3I3p6%+$i3+{p~GMeQ>=!+9yEsnn65EA$gpLm<~rPCbl7c_IIxaf(QBh6feLXAJqx!
zX`?r#Z=Va~LGVofyq>ncIGamRZrm04-Rii)Z0L2Q^#b$I@>9Qm;56|}t)kj<%007_
zNcn(y`yP1X*XYe>G-7#s_UdnSwvjx@-=ehLLey*Z!aEM?5~}xIu1I-;e$0=aE7ee4
z`|7PlTs&z&<~jBSFUud
zoR9Qlj@D|CS>6drLFykI?}DGx*6{ozeTHXfca3J2ei4NFzz6mTQC?dY+Y<0^t~*w;
z#mk1^ssEXuwt~n23!9{8N}HJb-`gUuAoT<0!JRp#(DPzh%CY*=DairHV@Mw4vpLfD
z_KM7rWMNVj)5q1JB1rQL{Fh{F#<(-JNgsM;Lpd&cI#40>503Y@+}`VR5p3;QBqTp<
z$>c8(vj6^i0nV@I$If%hU}WiRhcBevIzDfUhj^ylxhdPz*Emh2##@^=)L!8bJXl}J
zj&-X_2GP|ITOJ<*8=Y@AB6u*LamoT5uBrT)yRj0O)8>iXuzsEm4(+J$xT_31ZFP*X
z&AvqH2OKZ`k=V-flRK2__2*LBMY1K=?b?;FOY{pf*2H}GhP7Zp95vne&M
z0Xz!g)mnb~vz)n-1UwtWFKB3a?=PII>+F1g(`2&s2;uSH=K;*my<(Hqt`XgrZ_|^$
z9J>Efa2mmbd=5KTvK7Cdqwg9vUi~vB+t!ZYLH(`fUg6)29}kV#OkUqLp{2ngcrgE!
zQ&kRl4{sx*(&olH=|5|75Ioqg&PZfNrRB?)Eo{E7+utr^z(Ra$dFR7RwX7$!`_A-{
zbucg{B6!As_J?La&5HA176ZKp!@kd4mUzF6;6eW+6O75BJ!e`CNEf#>oyleJ2p;Sw
zHl)>mW;9VfGDljTi7@q~B6(2%gopg62MdCf+vn}(T_hX58WHLPU$991j%Lx_*Zs-p
z%Ov62*J8|K;@78muqXlD&+y+A*!
zV-4O>F^W#hv98^gx4YeGLR!LuUJ45qq$V}a4e)H>*z
zuoLr{6$}gr^}&9k+Ah~+|6#x2tkz1@hGkO6@}s}+SHP=XGwYv}A{L)KiRyE-jTZSYt!m|JAwztOK~r0ZmdLI{Z8qox|+qcWh4*se;M~_Ab+Li
z@ynl5m6xubrmjS&4}3s<&OQ3_V|1&cL$||~M7OXA9`sZA;%f`}%!#KRB`2i6(qx7p
zc~Bp}I`w6|B(*8Kbw)1L`H@m6LVa-kb7U5aeTToS4_;q#5Io4AE&5vOG8y~pt=QcV
zk=d8DNcBPf?GJIEd+GYthH5gbref*BFbMU*{FtuPp%Oa99+YZweX092XyT6G!SQN`
z_E@rat@^(!#$PL+t*|5PzWC6Q>*FWS>W^SZ982y`g@97sT6_-?>Fy!
zVJeo5FB7`?G55Wn+IL|M1b^h8*RQ?r@%tx&eRpZ+PX7q?@)$*$AF#fRPJ%lc@0y>^
zl3KUrPk2rstrxg{lVOM-x+=jTW^>k9;g3^9Y$8HGpgu?EsRGAjd@xT|^CqKi?VJyS
z2l+5O7%b#TrZ?UzfAt#U?Iega51@YO9l}$?ot*8OvpJb0`jnIK>YV)_VZ8i{
zS2KkT&bv~HAoK$~i-B{+3UTI4|9Fi;q(AOF6@mx(Xc*t;etQ&UwM}Zx(YwG{&4b{<
zJQv4R>80=Sr2Zg2tNHEMw=S+Dh85Nld4+{=R>L`lXkR>*vGW
zc2ry@bBHW&Z}%X0F#ps7^ZTvd`%?0igbvpz2gyc=-xK=ga6HU9JJ6lTBaFOh%`lAs%
z=qLY`U+tL3=HQ7;3T9{fE_pKq59%|g8hG;EZ7F=o;kk5F#n(d$;&Er&ByOGY#BgH6
zNa(cfw-_LPb?2!)i9#8h%upH`_tVpn%b9HYoc+!@UTzsP6DATbZ{Z_SCmJASonfU1|oDh}?
z_~neh&kyM5L(ZCC-^uDH!1qABH2#m*ZK1V{UngW`h*yuXRYLrDrDNL(f|x4si4b3B
zwg0}|;*TltJ`ity1$E0$-%=8IGl=iXPxKgm<{GbAGgMZDN+*(n_}(V5y`$)*3yTib
z_XYWkk26608TpH+^feecRe4_Us1x&NFQ*@z|H|{jFTZhLnsm%QHsE%1kkAA1PruP$
zOk*4ty?d@6BRzcnWhKO47M;M?{Bhx9r@d>xn`NS#h~UBb^=R#5;Qf
z&j#__cbVPQeh@JMPrx3WznWb5A-*)NL%`!9p6y22jU^dhTE^uz$;NH}_c(|jB&vVK
zf5qz}mrLJQW2e<2EX0R4m|P7`=Iplj6nk}y{AUgZ;zg43xF-GEoJR1Qb~oPM&_F}{
zb=7lkbMlw#h*69DUQ`()D2R8Ph@8-jCnqKqKJn2xdQy@N;?s+C9i^{bFwCVYU%o10
z(nEN7aQ=jnZoFk#2+g72ZFysc78-bne?wup;jS5cg^4WhX7(q?SR8@}?=OOgdKIpz
z@d~m8cXRg$omX!}@L-;M!jckfE6aR|n^t!{71M$S&Q*P9FXN8|-5Bq$jKN*E0
zc<_EY?xs7ht|Qe4{m*#aZkgBr
z6`NSipt6^#9`A%uAG|)6(2T}*G|Pq-81WYOQMLu55j?2>Ikxz+erA#%gM?jte?zbt
z3c-W@giJ)F`NU(7@K2@631!9J?|2YAsQ;|xm);;?o%Z*61OBzm{;E@lM2F>qHSnPj|D<2A)h3wD0eE+a&!P!8Cf3Zq3cMM_hmBiQUgYdQ
z2RsVmAM#yX^RX`F0iFlqaVH9!X`lFi1)d7xRfVeWJ&x4<0sL~x!Si51@!`~Swdoe%
z@er>cb!_nb3-KP{8zG)lt6Te{{6hiYiy>a%qne$^nouq9i4gyc*2p`V{24Xy7>Li@
z9=f;7l^FuO6U6T*q4rFkjkp7^2k~agyo078-#-E`1@U4IUrXNiYWO(
z%W;BFO0vV2Kjt8Ku)mpp6FtEyZP^WcD8ySllnO~7etgv+W-0m@;m8Xoi1!!>pDSG?l1O`Gap%-<_-gz7gW@u1Vf}
z^1A38@Hr5#VUXd7Rz}}L{la;#S`BxGLj1AV@R`GBU3-Cdg7`*(c{Nhhra$l+5WoG0
zu*J1lr{*5la?j&SL^BV>t4}O3nGfTS0#5|-XTA@gG@esn&YC#-eX_0L>saE!`PX04
zQCpuCzYlyP#4FygvyqmB;QWW@7k8dL^q2&`5#qTLZdE)Gd)P>i?dVjQW&DT92fsDH-rY5dVbl{(GNP9fhwJlyALvjwO!8|2=>3e&lf(-8ufMK2G4<
zA)ZJ&k@ftF(jDL{A%3IZ`FOLsTnX@4h_@O)Q&n|e<}L7{5Z@r4e&e=-zp(kX2j6p2s{A(Vyhxmy%w_W