ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-09-20 10:18:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

scalar chars refactor, some stuff got fixed, some stuff broke

Browse Source
This commit is contained in:
gluax 2021-06-11 16:16:21 -07:00
parent 0dbbf7496a
commit d1ca1108fb
24 changed files with 548 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
asg/src/const_value.rs
View File

@ -102,6 +102,22 @@ impl From<leo_ast::GroupValue> for GroupValue {
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum CharValue {
Scalar(char),
NonScalar(u32),
}
impl From<leo_ast::CharValue> for CharValue {
fn from(other: leo_ast::CharValue) -> Self {
use leo_ast::Char::*;
match other.character {
Scalar(value) => CharValue::Scalar(value),
NonScalar(value) => CharValue::NonScalar(value),
}
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum ConstValue {
Int(ConstInt),
@ -109,7 +125,7 @@ pub enum ConstValue {
Field(BigInt),
Address(StrTendril),
Boolean(bool),
Char(char),
Char(CharValue),
// compounds
Tuple(Vec<ConstValue>),

20
asg/src/expression/constant.rs
View File

@ -16,6 +16,7 @@
use crate::{
AsgConvertError,
CharValue,
ConstInt,
ConstValue,
Expression,
@ -118,22 +119,22 @@ impl<'a> FromAst<'a, leo_ast::ValueExpression> for Constant<'a> {
),
}
}
Char(value, span) => {
Char(value) => {
match expected_type.map(PartialType::full).flatten() {
Some(Type::Char) | None => (),
Some(x) => {
return Err(AsgConvertError::unexpected_type(
&x.to_string(),
Some(&*Type::Char.to_string()),
span,
value.span(),
));
}
}
Constant {
parent: Cell::new(None),
span: Some(span.clone()),
value: ConstValue::Char(*value),
span: Some(value.span().clone()),
value: ConstValue::Char(CharValue::from(value.clone())),
}
}
Field(value, span) => {
@ -236,7 +237,16 @@ impl<'a> Into<leo_ast::ValueExpression> for &Constant<'a> {
ConstValue::Boolean(value) => {
leo_ast::ValueExpression::Boolean(value.to_string().into(), self.span.clone().unwrap_or_default())
}
ConstValue::Char(value) => leo_ast::ValueExpression::Char(*value, self.span.clone().unwrap_or_default()),
ConstValue::Char(value) => match value {
CharValue::Scalar(scalar) => leo_ast::ValueExpression::Char(leo_ast::CharValue {
character: leo_ast::Char::Scalar(*scalar),
span: self.span.clone().unwrap_or_default(),
}),
CharValue::NonScalar(non_scalar) => leo_ast::ValueExpression::Char(leo_ast::CharValue {
character: leo_ast::Char::NonScalar(*non_scalar),
span: self.span.clone().unwrap_or_default(),
}),
},
ConstValue::Field(value) => {
leo_ast::ValueExpression::Field(value.to_string().into(), self.span.clone().unwrap_or_default())
}

57
ast/src/chars/char_value.rs Normal file
View File

@ -0,0 +1,57 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::common::span::Span;
use serde::{Deserialize, Serialize};
use std::fmt;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Char {
Scalar(char),
NonScalar(u32),
}
impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Scalar(c) => write!(f, "{}", c),
Self::NonScalar(c) => write!(f, "{}", c),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct CharValue {
pub character: Char,
pub span: Span,
}
impl fmt::Display for CharValue {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.character)
}
}
impl CharValue {
pub fn set_span(&mut self, new_span: Span) {
self.span = new_span;
}
pub fn span(&self) -> &Span {
&self.span
}
}

18
ast/src/chars/mod.rs Normal file
View File

@ -0,0 +1,18 @@
// Copyright (C) 2019-2021 Aleo Systems Inc.
// This file is part of the Leo library.
// The Leo library is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// The Leo library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
pub mod char_value;
pub use self::char_value::*;

19
ast/src/expression/value.rs
View File

@ -17,14 +17,14 @@
use tendril::StrTendril;
use super::*;
use crate::GroupTuple;
use crate::{Char, CharValue, GroupTuple};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum ValueExpression {
// todo: deserialize values here
Address(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Boolean(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Char(char, Span),
Char(CharValue),
Field(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Group(Box<GroupValue>),
Implicit(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
@ -33,7 +33,7 @@ pub enum ValueExpression {
#[serde(with = "crate::common::tendril_json")] StrTendril,
Span,
),
String(String, Span),
String(Vec<Char>, Span),
}
impl fmt::Display for ValueExpression {
@ -42,12 +42,17 @@ impl fmt::Display for ValueExpression {
match &self {
Address(address, _) => write!(f, "{}", address),
Boolean(boolean, _) => write!(f, "{}", boolean),
Char(character, _) => write!(f, "{}", character),
Char(character) => write!(f, "{}", character),
Field(field, _) => write!(f, "{}", field),
Implicit(implicit, _) => write!(f, "{}", implicit),
Integer(value, type_, _) => write!(f, "{}{}", value, type_),
Group(group) => write!(f, "{}", group),
String(string, _) => write!(f, "{}", string),
String(string, _) => {
for character in string.iter() {
write!(f, "{}", character)?;
}
Ok(())
}
}
}
}
@ -58,11 +63,11 @@ impl Node for ValueExpression {
match &self {
Address(_, span)
| Boolean(_, span)
| Char(_, span)
| Field(_, span)
| Implicit(_, span)
| Integer(_, _, span)
| String(_, span) => span,
Char(character) => &character.span,
Group(group) => match &**group {
GroupValue::Single(_, span) | GroupValue::Tuple(GroupTuple { span, .. }) => span,
},
@ -74,11 +79,11 @@ impl Node for ValueExpression {
match self {
Address(_, span)
| Boolean(_, span)
| Char(_, span)
| Field(_, span)
| Implicit(_, span)
| Integer(_, _, span)
| String(_, span) => *span = new_span,
Char(character) => character.span = new_span,
Group(group) => match &mut **group {
GroupValue::Single(_, span) | GroupValue::Tuple(GroupTuple { span, .. }) => *span = new_span,
},

19
ast/src/input/input_value.rs
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::{ArrayDimensions, GroupValue};
use crate::{ArrayDimensions, Char, CharValue, GroupValue, Span as AstSpan};
use leo_input::{
errors::InputParserError,
expressions::{ArrayInitializerExpression, ArrayInlineExpression, Expression, StringExpression, TupleExpression},
@ -23,7 +23,7 @@ use leo_input::{
Address,
AddressValue,
BooleanValue,
CharValue,
CharValue as InputCharValue,
FieldValue,
GroupValue as InputGroupValue,
NumberValue,
@ -38,7 +38,7 @@ use std::fmt;
pub enum InputValue {
Address(String),
Boolean(bool),
Char(char),
Char(CharValue),
Field(String),
Group(GroupValue),
Integer(IntegerType, String),
@ -63,9 +63,14 @@ impl InputValue {
Ok(InputValue::Boolean(boolean))
}
fn from_char(character: CharValue) -> Result<Self, InputParserError> {
let character = character.value.inner()?;
Ok(InputValue::Char(character))
fn from_char(input_character: InputCharValue) -> Result<Self, InputParserError> {
let character = match input_character.value.inner()? {
leo_input::values::Char::Scalar(scalar) => Char::Scalar(scalar),
leo_input::values::Char::NonScalar(non_scalar) => Char::NonScalar(non_scalar),
};
let span = AstSpan::from(input_character.span);
Ok(InputValue::Char(CharValue { character, span }))
}
fn from_number(integer_type: IntegerType, number: String) -> Self {
@ -157,7 +162,7 @@ impl InputValue {
for character in string.chars.into_iter() {
let element = InputValue::from_expression(
inner_array_type.clone(),
Expression::Value(Value::Char(CharValue {
Expression::Value(Value::Char(InputCharValue {
value: character.clone(),
span: character.span().clone(),
})),

3
ast/src/lib.rs
View File

@ -29,6 +29,9 @@ pub use self::annotation::*;
pub mod circuits;
pub use self::circuits::*;
pub mod chars;
pub use self::chars::*;
pub mod common;
pub use self::common::*;

9
ast/src/reducer/canonicalization.rs
View File

@ -485,11 +485,14 @@ impl ReconstructingReducer for Canonicalizer {
}
}
fn reduce_string(&mut self, string: &str, span: &Span) -> Result<Expression, ReducerError> {
fn reduce_string(&mut self, string: &[Char], span: &Span) -> Result<Expression, ReducerError> {
let mut elements = Vec::new();
for character in string.chars() {
for character in string {
elements.push(SpreadOrExpression::Expression(Expression::Value(
ValueExpression::Char(character, span.clone()),
ValueExpression::Char(CharValue {
character: character.clone(),
span: span.clone(),
}),
)));
}

2
ast/src/reducer/reconstructing_director.rs
View File

@ -100,7 +100,7 @@ impl<R: ReconstructingReducer> ReconstructingDirector<R> {
self.reducer.reduce_group_value(group_value, new)
}
pub fn reduce_string(&mut self, string: &str, span: &Span) -> Result<Expression, ReducerError> {
pub fn reduce_string(&mut self, string: &[Char], span: &Span) -> Result<Expression, ReducerError> {
self.reducer.reduce_string(string, span)
}

4
ast/src/reducer/reconstructing_reducer.rs
View File

@ -51,9 +51,9 @@ pub trait ReconstructingReducer {
Ok(new)
}
fn reduce_string(&mut self, string: &str, span: &Span) -> Result<Expression, ReducerError> {
fn reduce_string(&mut self, string: &[Char], span: &Span) -> Result<Expression, ReducerError> {
Ok(Expression::Value(ValueExpression::String(
string.to_string(),
string.to_vec(),
span.clone(),
)))
}

39
ast/src/statements/console/formatted_string.rs
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::{Expression, Node, Span};
use crate::{Char, Expression, Node, Span};
use serde::{Deserialize, Serialize};
use std::fmt;
@ -27,25 +27,34 @@ pub enum FormatStringPart {
}
impl FormatStringPart {
pub fn from_string(string: String) -> Vec<Self> {
pub fn from_string(string: Vec<Char>) -> Vec<Self> {
let mut parts = Vec::new();
let mut in_container = false;
let mut start = 0;
for (index, character) in string.chars().enumerate() {
// let mut start = 0;
let mut substring = String::new();
for (_, character) in string.iter().enumerate() {
match character {
'{' if !in_container => {
parts.push(FormatStringPart::Const(string[start..index].into()));
start = index;
in_container = true;
}
'}' if in_container => {
in_container = false;
parts.push(FormatStringPart::Container);
}
_ if in_container => {
Char::Scalar(scalar) => match scalar {
'{' if !in_container => {
// let subsection: Vec<Char> = string[start..index].to_vec();
parts.push(FormatStringPart::Const(substring.clone().into()));
// start = index;
substring.clear();
in_container = true;
}
'}' if in_container => {
in_container = false;
parts.push(FormatStringPart::Container);
}
_ if in_container => {
in_container = false;
}
_ => substring.push(*scalar),
},
Char::NonScalar(non_scalar) => {
parts.push(FormatStringPart::Const(format!("\\u{{{:X}}}", non_scalar).into()));
in_container = false;
}
_ => {}
}
}

18
compiler/src/expression/expression.rs
View File

@ -23,7 +23,7 @@ use crate::{
program::ConstrainedProgram,
relational::*,
resolve_core_circuit,
value::{Address, Char, ConstrainedValue, Integer},
value::{Address, Char, CharType, ConstrainedValue, Integer},
FieldType,
GroupType,
};
@ -44,7 +44,21 @@ impl<'a, F: PrimeField, G: GroupType<F>> ConstrainedProgram<'a, F, G> {
ConstValue::Address(value) => ConstrainedValue::Address(Address::constant(value.to_string(), span)?),
ConstValue::Boolean(value) => ConstrainedValue::Boolean(Boolean::Constant(*value)),
ConstValue::Char(value) => {
ConstrainedValue::Char(Char::constant(cs, *value, format!("{}", *value as u32), span)?)
use leo_asg::CharValue::*;
match value {
Scalar(scalar) => ConstrainedValue::Char(Char::constant(
cs,
CharType::Scalar(*scalar),
format!("{}", *scalar as u32),
span,
)?),
NonScalar(non_scalar) => ConstrainedValue::Char(Char::constant(
cs,
CharType::NonScalar(*non_scalar),
format!("{}", *non_scalar),
span,
)?),
}
}
ConstValue::Field(value) => ConstrainedValue::Field(FieldType::constant(cs, value.to_string(), span)?),
ConstValue::Group(value) => ConstrainedValue::Group(G::constant(value, span)?),

22
compiler/src/function/input/main_function_input.rs
View File

@ -27,13 +27,13 @@ use crate::{
group::input::group_from_input,
ConstrainedValue,
},
Char,
CharType,
FieldType,
GroupType,
Integer,
};
use leo_asg::{ConstInt, Type};
use leo_ast::{InputValue, Span};
use leo_ast::{Char, InputValue, Span};
use snarkvm_fields::PrimeField;
use snarkvm_gadgets::boolean::Boolean;
@ -83,12 +83,18 @@ impl<'a, F: PrimeField, G: GroupType<F>> ConstrainedProgram<'a, F, G> {
match (type_, input) {
(Type::Address, InputValue::Address(addr)) => Ok(ConstrainedValue::Address(Address::constant(addr, span)?)),
(Type::Boolean, InputValue::Boolean(value)) => Ok(ConstrainedValue::Boolean(Boolean::constant(value))),
(Type::Char, InputValue::Char(character)) => Ok(ConstrainedValue::Char(Char::constant(
cs,
character,
format!("{}", character as u32),
span,
)?)),
(Type::Char, InputValue::Char(character)) => {
let c = match character.character {
Char::Scalar(scalar) => CharType::Scalar(scalar),
Char::NonScalar(non_scalar) => CharType::NonScalar(non_scalar),
};
Ok(ConstrainedValue::Char(crate::Char::constant(
cs,
c,
format!("{}", character),
span,
)?))
}
(Type::Field, InputValue::Field(value)) => {
Ok(ConstrainedValue::Field(FieldType::constant(cs, value, span)?))
}

10
compiler/src/phases/reducing_director.rs
View File

@ -65,6 +65,8 @@ use leo_ast::{
Block as AstBlockStatement,
CallExpression as AstCallExpression,
CastExpression as AstCastExpression,
Char,
CharValue,
Circuit as AstCircuit,
CircuitImpliedVariableDefinition,
CircuitInitExpression as AstCircuitInitExpression,
@ -432,9 +434,13 @@ impl<R: ReconstructingReducer, O: CombinerOptions> CombineAstAsgDirector<R, O> {
new = ValueExpression::Boolean(tendril.clone(), span.clone());
}
ConstValue::Char(_) => {
if let Some(c) = tendril.chars().next() {
new = ValueExpression::Char(c, span.clone());
if let Some(character) = tendril.chars().next() {
new = ValueExpression::Char(CharValue {
character: Char::Scalar(character),
span: span.clone(),
});
} else {
// TODO handle implicit non-scalar
return Err(ReducerError::failed_to_convert_tendril_to_char(
tendril.to_string(),
span,

28
compiler/src/value/char/char.rs
View File

@ -31,17 +31,23 @@ use snarkvm_gadgets::{
};
use snarkvm_r1cs::{ConstraintSystem, SynthesisError};
#[derive(Clone, Debug)]
pub enum CharType {
Scalar(char),
NonScalar(u32),
}
/// A char
#[derive(Clone, Debug)]
pub struct Char<F: PrimeField> {
pub character: char,
pub character: CharType,
pub field: FieldType<F>,
}
impl<F: PrimeField> Char<F> {
pub fn constant<CS: ConstraintSystem<F>>(
cs: CS,
character: char,
character: CharType,
field: String,
span: &Span,
) -> Result<Self, CharError> {
@ -118,13 +124,13 @@ impl<F: PrimeField> CondSelectGadget<F> for Char<F> {
if field == first.field {
return Ok(Char {
character: first.character,
character: first.character.clone(),
field,
});
}
Ok(Char {
character: second.character,
character: second.character.clone(),
field,
})
}
@ -144,12 +150,17 @@ pub(crate) fn char_from_input<'a, F: PrimeField, G: GroupType<F>, CS: Constraint
let option = match input_value {
Some(input) => {
if let InputValue::Char(character) = input {
(character, Some((character as u32).to_string()))
match character.character {
leo_ast::Char::Scalar(scalar) => (CharType::Scalar(scalar), Some((scalar as u32).to_string())),
leo_ast::Char::NonScalar(non_scalar) => {
(CharType::NonScalar(non_scalar), Some(non_scalar.to_string()))
}
}
} else {
return Err(CharError::invalid_char(input.to_string(), span));
}
}
None => (' ', None),
None => (CharType::Scalar(0 as char), None),
};
let field = allocate_field(cs, name, option.1, span)?;
@ -162,6 +173,9 @@ pub(crate) fn char_from_input<'a, F: PrimeField, G: GroupType<F>, CS: Constraint
impl<F: PrimeField + std::fmt::Display> std::fmt::Display for Char<F> {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(f, "{}", char::escape_default(self.character))
match self.character {
CharType::Scalar(scalar) => write!(f, "{}", char::escape_default(scalar)),
CharType::NonScalar(non_scalar) => write!(f, "\\u{{{:X}}}", non_scalar),
}
}
}

224
compiler/tests/canonicalization/string_transformation.json
View File

@ -20,248 +20,284 @@
"variable_names": [
{
"mutable": true,
"identifier": "{\"name\":\"s\",\"span\":\"{\\\"line_start\\\":2,\\\"line_stop\\\":2,\\\"col_start\\\":9,\\\"col_stop\\\":10,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" let s = \\\\\\\"Hello, World!\\\\\\\";\\\"}\"}",
"identifier": "{\"name\":\"s\",\"span\":\"{\\\"line_start\\\":2,\\\"line_stop\\\":2,\\\"col_start\\\":9,\\\"col_stop\\\":10,\\\"path\\\":\\\"\\\",\\\"content\\\":\\\" let s = \\\\\\\"Hello, World!\\\\\\\\u{DDDD}\\\\\\\";\\\"}\"}",
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 9,
"col_stop": 10,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
}
],
"type_": {
"Array": [
"Char",
[
{
"value": "13"
}
]
]
},
"type_": null,
"value": {
"ArrayInline": {
"elements": [
{
"Expression": {
"Value": {
"Char": [
"H",
{
"Char": {
"character": {
"Scalar": "H"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"e",
{
"Char": {
"character": {
"Scalar": "e"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"l",
{
"Char": {
"character": {
"Scalar": "l"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"l",
{
"Char": {
"character": {
"Scalar": "l"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"o",
{
"Char": {
"character": {
"Scalar": "o"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
",",
{
"Char": {
"character": {
"Scalar": ","
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
" ",
{
"Char": {
"character": {
"Scalar": " "
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"W",
{
"Char": {
"character": {
"Scalar": "W"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"o",
{
"Char": {
"character": {
"Scalar": "o"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"r",
{
"Char": {
"character": {
"Scalar": "r"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"l",
{
"Char": {
"character": {
"Scalar": "l"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"d",
{
"Char": {
"character": {
"Scalar": "d"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": [
"!",
{
"Char": {
"character": {
"Scalar": "!"
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
]
}
}
}
},
{
"Expression": {
"Value": {
"Char": {
"character": {
"NonScalar": 56797
},
"span": {
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
}
}
}
}
@ -270,9 +306,9 @@
"line_start": 2,
"line_stop": 2,
"col_start": 13,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
}
},
@ -280,9 +316,9 @@
"line_start": 2,
"line_stop": 2,
"col_start": 5,
"col_stop": 28,
"col_stop": 36,
"path": "",
"content": " let s = \"Hello, World!\";"
"content": " let s = \"Hello, World!\\u{DDDD}\";"
}
}
}

2
compiler/tests/canonicalization/string_transformation.leo
View File

@ -1,3 +1,3 @@
function main() {
let s = "Hello, World!";
let s = "Hello, World!\u{DDDD}";
}

30
input/src/values/char_types.rs
View File

@ -78,12 +78,18 @@ impl<'ast> CharTypes<'ast> {
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Char {
Scalar(char),
NonScalar(u32),
}
impl<'ast> CharTypes<'ast> {
pub fn inner(self) -> Result<char, InputParserError> {
pub fn inner(self) -> Result<Char, InputParserError> {
match self {
Self::Basic(character) => {
if let Some(character) = character.value.chars().next() {
return Ok(character);
return Ok(Char::Scalar(character));
}
Err(InputParserError::invalid_char(character.value, &character.span))
@ -91,13 +97,13 @@ impl<'ast> CharTypes<'ast> {
Self::Escaped(character) => {
if let Some(inner) = character.value.chars().nth(1) {
return match inner {
'0' => Ok(0 as char),
't' => Ok(9 as char),
'n' => Ok(10 as char),
'r' => Ok(13 as char),
'\"' => Ok(34 as char),
'\'' => Ok(39 as char),
'\\' => Ok(92 as char),
'0' => Ok(Char::Scalar(0 as char)),
't' => Ok(Char::Scalar(9 as char)),
'n' => Ok(Char::Scalar(10 as char)),
'r' => Ok(Char::Scalar(13 as char)),
'\"' => Ok(Char::Scalar(34 as char)),
'\'' => Ok(Char::Scalar(39 as char)),
'\\' => Ok(Char::Scalar(92 as char)),
_ => Err(InputParserError::invalid_char(character.value, &character.span)),
};
}
@ -108,7 +114,7 @@ impl<'ast> CharTypes<'ast> {
let hex_string_number = character.value[2..character.value.len()].to_string();
if let Ok(number) = u8::from_str_radix(&hex_string_number, 16) {
if number < 127 {
return Ok(number as char);
return Ok(Char::Scalar(number as char));
}
}
@ -118,7 +124,9 @@ impl<'ast> CharTypes<'ast> {
let unicode_string_number = character.value[3..=character.value.len() - 2].to_string();
if let Ok(hex) = u32::from_str_radix(&unicode_string_number, 16) {
if let Some(unicode) = std::char::from_u32(hex) {
return Ok(unicode);
return Ok(Char::Scalar(unicode));
} else if hex <= 0x10FFFF {
return Ok(Char::NonScalar(hex));
}
}

5
parser/src/parser/expression.rs
View File

@ -689,7 +689,10 @@ impl ParserContext {
Token::True => Expression::Value(ValueExpression::Boolean("true".into(), span)),
Token::False => Expression::Value(ValueExpression::Boolean("false".into(), span)),
Token::AddressLit(value) => Expression::Value(ValueExpression::Address(value, span)),
Token::CharLit(value) => Expression::Value(ValueExpression::Char(value, span)),
Token::CharLit(value) => Expression::Value(ValueExpression::Char(CharValue {
character: value.into(),
span,
})),
Token::StringLit(value) => Expression::Value(ValueExpression::String(value, span)),
Token::LeftParen => self.parse_tuple_expression(&span)?,
Token::LeftSquare => self.parse_array_expression(&span)?,

31
parser/src/tokenizer/lexer.rs
View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU General Public License
// along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
use crate::tokenizer::Token;
use crate::tokenizer::{Char, Token};
use leo_ast::Span;
use serde::{Deserialize, Serialize};
use tendril::StrTendril;
@ -64,7 +64,7 @@ impl Token {
///
/// Returns a `char` if an character can be eaten, otherwise returns [`None`].
///
fn eat_char(input_tendril: StrTendril, escaped: bool, hex: bool, unicode: bool) -> Option<char> {
fn eat_char(input_tendril: StrTendril, escaped: bool, hex: bool, unicode: bool) -> Option<Char> {
if input_tendril.is_empty() {
return None;
}
@ -79,13 +79,13 @@ impl Token {
if let Some(character) = escaped.chars().next() {
return match character {
'0' => Some(0 as char),
't' => Some(9 as char),
'n' => Some(10 as char),
'r' => Some(13 as char),
'\"' => Some(34 as char),
'\'' => Some(39 as char),
'\\' => Some(92 as char),
'0' => Some(Char::Scalar(0 as char)),
't' => Some(Char::Scalar(9 as char)),
'n' => Some(Char::Scalar(10 as char)),
'r' => Some(Char::Scalar(13 as char)),
'\"' => Some(Char::Scalar(34 as char)),
'\'' => Some(Char::Scalar(39 as char)),
'\\' => Some(Char::Scalar(92 as char)),
_ => None,
};
} else {
@ -107,7 +107,7 @@ impl Token {
return None;
}
return Some(ascii_number as char);
return Some(Char::Scalar(ascii_number as char));
}
}
@ -121,7 +121,10 @@ impl Token {
if let Ok(hex) = u32::from_str_radix(&unicode_number, 16) {
if let Some(character) = std::char::from_u32(hex) {
return Some(character);
// scalar
return Some(Char::Scalar(character));
} else if hex <= 0x10FFFF {
return Some(Char::NonScalar(hex));
}
}
}
@ -129,7 +132,7 @@ impl Token {
if input_tendril.to_string().chars().count() != 1 {
return None;
} else if let Some(character) = input_tendril.to_string().chars().next() {
return Some(character);
return Some(Char::Scalar(character));
}
None
@ -205,7 +208,7 @@ impl Token {
let mut hex = false;
let mut unicode = false;
let mut end = false;
let mut string = String::new();
let mut string = Vec::new();
while i < input.len() {
// If it's an emoji get the length.
@ -260,7 +263,7 @@ impl Token {
escaped = false;
hex = false;
unicode = false;
string.push(character);
string.push(character.into());
}
None => return (0, None),
}

37
parser/src/tokenizer/token.rs
View File

@ -18,6 +18,31 @@ use serde::{Deserialize, Serialize};
use std::fmt;
use tendril::StrTendril;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Char {
Scalar(char),
NonScalar(u32),
}
#[allow(clippy::from_over_into)]
impl Into<leo_ast::Char> for Char {
fn into(self) -> leo_ast::Char {
match self {
Self::Scalar(c) => leo_ast::Char::Scalar(c),
Self::NonScalar(c) => leo_ast::Char::NonScalar(c),
}
}
}
impl fmt::Display for Char {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Scalar(c) => write!(f, "{}", c),
Self::NonScalar(c) => write!(f, "{}", c),
}
}
}
/// Represents all valid Leo syntax tokens.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Token {
@ -25,13 +50,13 @@ pub enum Token {
// Literals
CommentLine(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
CommentBlock(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
StringLit(String),
StringLit(Vec<leo_ast::Char>),
Ident(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
Int(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
True,
False,
AddressLit(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
CharLit(char),
CharLit(Char),
At,
@ -191,7 +216,13 @@ impl fmt::Display for Token {
match self {
CommentLine(s) => write!(f, "{}", s),
CommentBlock(s) => write!(f, "{}", s),
StringLit(content) => write!(f, "\"{}\"", content),
StringLit(string) => {
write!(f, "\"")?;
for character in string.iter() {
write!(f, "{}", character)?;
}
write!(f, "\"")
}
Ident(s) => write!(f, "{}", s),
Int(s) => write!(f, "{}", s),
True => write!(f, "true"),

66
tests/expectations/parser/parser/expression/literal/char_parse.leo.out
View File

@ -4,8 +4,10 @@ expectation: Pass
outputs:
- Value:
Char:
- a
- line_start: 1
character:
Scalar: a
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 4
@ -13,8 +15,10 @@ outputs:
content: "'a'"
- Value:
Char:
- Z
- line_start: 1
character:
Scalar: Z
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 4
@ -22,8 +26,10 @@ outputs:
content: "'Z'"
- Value:
Char:
- "\""
- line_start: 1
character:
Scalar: "\""
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 5
@ -31,8 +37,10 @@ outputs:
content: "'\\\"'"
- Value:
Char:
- "\t"
- line_start: 1
character:
Scalar: "\t"
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 5
@ -40,8 +48,10 @@ outputs:
content: "'\\t'"
- Value:
Char:
- "\r"
- line_start: 1
character:
Scalar: "\r"
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 5
@ -49,8 +59,10 @@ outputs:
content: "'\\r'"
- Value:
Char:
- "\u0000"
- line_start: 1
character:
Scalar: "\u0000"
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 5
@ -58,8 +70,10 @@ outputs:
content: "'\\0'"
- Value:
Char:
- ❤
- line_start: 1
character:
Scalar: ❤
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 11
@ -67,8 +81,10 @@ outputs:
content: "'\\u{2764}'"
- Value:
Char:
- の
- line_start: 1
character:
Scalar: の
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 11
@ -76,8 +92,10 @@ outputs:
content: "'\\u{306E}'"
- Value:
Char:
- ❤
- line_start: 1
character:
Scalar: ❤
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 6
@ -85,8 +103,10 @@ outputs:
content: "'❤'"
- Value:
Char:
- の
- line_start: 1
character:
Scalar: の
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 6
@ -94,8 +114,10 @@ outputs:
content: "'の'"
- Value:
Char:
- "*"
- line_start: 1
character:
Scalar: "*"
span:
line_start: 1
line_stop: 1
col_start: 1
col_stop: 7

84
tests/expectations/parser/parser/expression/literal/string_parse.leo.out
View File

@ -4,7 +4,12 @@ expectation: Pass
outputs:
- Value:
String:
- string
- - Scalar: s
- Scalar: t
- Scalar: r
- Scalar: i
- Scalar: n
- Scalar: g
- line_start: 1
line_stop: 1
col_start: 1
@ -13,7 +18,24 @@ outputs:
content: "\"string\""
- Value:
String:
- "another { } string"
- - Scalar: a
- Scalar: n
- Scalar: o
- Scalar: t
- Scalar: h
- Scalar: e
- Scalar: r
- Scalar: " "
- Scalar: "{"
- Scalar: " "
- Scalar: "}"
- Scalar: " "
- Scalar: s
- Scalar: t
- Scalar: r
- Scalar: i
- Scalar: n
- Scalar: g
- line_start: 1
line_stop: 1
col_start: 1
@ -22,7 +44,15 @@ outputs:
content: "\"another { } string\""
- Value:
String:
- "{ ] [ ; a"
- - Scalar: "{"
- Scalar: " "
- Scalar: "]"
- Scalar: " "
- Scalar: "["
- Scalar: " "
- Scalar: ;
- Scalar: " "
- Scalar: a
- line_start: 1
line_stop: 1
col_start: 1
@ -31,7 +61,7 @@ outputs:
content: "\"{ ] [ ; a\""
- Value:
String:
- ࿺
- - Scalar: ࿺
- line_start: 1
line_stop: 1
col_start: 1
@ -40,7 +70,7 @@ outputs:
content: "\"\\u{FFA}\""
- Value:
String:
- 򯫺
- - Scalar: 򯫺
- line_start: 1
line_stop: 1
col_start: 1
@ -49,7 +79,7 @@ outputs:
content: "\"\\u{afafa}\""
- Value:
String:
- 꾯
- - Scalar: 꾯
- line_start: 1
line_stop: 1
col_start: 1
@ -58,7 +88,7 @@ outputs:
content: "\"\\u{afaf}\""
- Value:
String:
- ૺ
- - Scalar: ૺ
- line_start: 1
line_stop: 1
col_start: 1
@ -67,7 +97,7 @@ outputs:
content: "\"\\u{afa}\""
- Value:
String:
- ¯
- - Scalar: ¯
- line_start: 1
line_stop: 1
col_start: 1
@ -76,7 +106,7 @@ outputs:
content: "\"\\u{af}\""
- Value:
String:
- "\n"
- - Scalar: "\n"
- line_start: 1
line_stop: 1
col_start: 1
@ -85,7 +115,7 @@ outputs:
content: "\"\\u{a}\""
- Value:
String:
- "\n"
- - Scalar: "\n"
- line_start: 1
line_stop: 1
col_start: 1
@ -94,7 +124,7 @@ outputs:
content: "\"\\x0A\""
- Value:
String:
- 
- - Scalar: 
- line_start: 1
line_stop: 1
col_start: 1
@ -103,7 +133,25 @@ outputs:
content: "\"\\x7F\""
- Value:
String:
- "aa \\ \" \n aa \t \r \u0000"
- - Scalar: a
- Scalar: a
- Scalar: " "
- Scalar: "\\"
- Scalar: " "
- Scalar: "\""
- Scalar: " "
- Scalar: " "
- Scalar: "\n"
- Scalar: " "
- Scalar: a
- Scalar: a
- Scalar: " "
- Scalar: "\t"
- Scalar: " "
- Scalar: "\r"
- Scalar: " "
- Scalar: " "
- Scalar: "\u0000"
- line_start: 1
line_stop: 1
col_start: 1
@ -112,7 +160,13 @@ outputs:
content: "\"aa \\\\ \\\" \\n aa \\t \\r \\0\""
- Value:
String:
- test 😒€
- - Scalar: t
- Scalar: e
- Scalar: s
- Scalar: t
- Scalar: " "
- Scalar: 😒
- Scalar: €
- line_start: 1
line_stop: 1
col_start: 1
@ -121,7 +175,9 @@ outputs:
content: "\"test 😒€\""
- Value:
String:
- 😭😂😘
- - Scalar: 😭
- Scalar: 😂
- Scalar: 😘
- line_start: 1
line_stop: 1
col_start: 1

6
tests/expectations/parser/parser/statement/console.leo.out
View File

@ -41,7 +41,7 @@ outputs:
parts:
- Const: ""
- Container
- Const: "{}"
- Const: ""
- Container
parameters:
- Identifier: "{\"name\":\"x\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":23,\\\"col_stop\\\":24,\\\"path\\\":\\\"test\\\",\\\"content\\\":\\\"console.error(\\\\\\\"{}{}\\\\\\\", x, y);\\\"}\"}"
@ -107,7 +107,7 @@ outputs:
parts:
- Const: ""
- Container
- Const: "{}"
- Const: ""
- Container
parameters:
- Identifier: "{\"name\":\"x\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":23,\\\"col_stop\\\":24,\\\"path\\\":\\\"test\\\",\\\"content\\\":\\\"console.debug(\\\\\\\"{}{}\\\\\\\", x, y);\\\"}\"}"
@ -173,7 +173,7 @@ outputs:
parts:
- Const: ""
- Container
- Const: "{}"
- Const: ""
- Container
parameters:
- Identifier: "{\"name\":\"x\",\"span\":\"{\\\"line_start\\\":1,\\\"line_stop\\\":1,\\\"col_start\\\":21,\\\"col_stop\\\":22,\\\"path\\\":\\\"test\\\",\\\"content\\\":\\\"console.log(\\\\\\\"{}{}\\\\\\\", x, y);\\\"}\"}"