AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2024-11-28 19:23:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

addex hex, octal, and direct unicode

Browse Source
This commit is contained in:
gluax 2021-05-13 14:33:01 -04:00
parent 29cd920075
commit 6e71c8ab4f
7 changed files with 190 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
asg/src/expression/constant.rs
View File

@ -129,14 +129,11 @@ impl<'a> FromAst<'a, leo_ast::ValueExpression> for Constant<'a> {
));
}
}
Constant {
parent: Cell::new(None),
span: Some(span.clone()),
value: ConstValue::Char(
value
.parse::<char>()
.map_err(|_| AsgConvertError::invalid_char(&value, span))?,
),
value: ConstValue::Char(*value),
}
}
Field(value, span) => {
@ -237,7 +234,7 @@ impl<'a> Into<leo_ast::ValueExpression> for &Constant<'a> {
leo_ast::ValueExpression::Boolean(value.to_string().into(), self.span.clone().unwrap_or_default())
}
ConstValue::Char(value) => {
leo_ast::ValueExpression::Char(value.to_string().into(), self.span.clone().unwrap_or_default())
leo_ast::ValueExpression::Char(value.clone(), self.span.clone().unwrap_or_default())
}
ConstValue::Field(value) => {
leo_ast::ValueExpression::Field(value.to_string().into(), self.span.clone().unwrap_or_default())

2
ast/src/expression/value.rs
View File

@ -24,7 +24,7 @@ pub enum ValueExpression {
// todo: deserialize values here
Address(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Boolean(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Char(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Char(char, Span),
Field(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),
Group(Box<GroupValue>),
Implicit(#[serde(with = "crate::common::tendril_json")] StrTendril, Span),

5
compiler/src/phases/reducing_director.rs
View File

@ -434,7 +434,10 @@ impl<R: ReconstructingReducer, O: CombinerOptions> CombineAstAsgDirector<R, O> {
new = ValueExpression::Boolean(tendril.clone(), span.clone());
}
ConstValue::Char(_) => {
new = ValueExpression::Char(tendril.clone(), span.clone());
if let Some(c) = tendril.chars().next() {
new = ValueExpression::Char(c, span.clone());
}
// TODO RETURN ERR
}
_ => unimplemented!(), // impossible?
}

189
parser/src/tokenizer/lexer.rs
View File

@ -151,53 +151,184 @@ impl Token {
return (i + 1, Some(Token::FormatString(segments)));
}
b'\'' => {
if input[1] == b'\'' {
return (0, None);
}
let mut i = 1;
let mut in_escape = false;
let mut character = String::new();
let mut escaped = false;
let mut hex = false;
let mut octal = false;
let mut unicode = true;
let mut characters: Vec<u8> = vec![];
while i < input.len() {
if !in_escape {
if !escaped {
if input[i] == b'\'' {
i += 1;
break;
}
if input[i] == b'\\' {
in_escape = !in_escape;
} else {
character.push(input[i] as char);
if input[i] == b'{' {
i += 1;
characters.clear();
continue;
}
if input[i] == b'}' {
i += 1;
continue;
}
} else {
in_escape = false;
if input[i] == b'u' {
i += 2;
let mut j = i;
let mut size = 0;
while input[j] != b'}' {
j += 1;
size += 1;
}
let hex_string_number: String = input_tendril.subtendril(i as u32, size).to_string();
if let Ok(hex) = u32::from_str_radix(&hex_string_number, 16) {
if let Some(unicode) = std::char::from_u32(hex) {
i = j;
character = unicode.to_string();
escaped = false;
characters.clear();
match input[i] {
b'0' => characters.push(0),
b't' => characters.push(9),
b'n' => characters.push(10),
b'r' => characters.push(13),
b'\"' => characters.push(34),
b'\'' => characters.push(39),
b'\\' => characters.push(92),
b'x' => {
i += 1;
match input[i] {
b'H' => {
hex = true;
}
b'O' => {
octal = true;
}
_ => {
return (0, None);
}
}
} else {
i += 1;
continue;
}
b'u' => {
unicode = true;
}
_ => {
return (0, None);
}
} else {
character.push(input[i] as char);
}
i += 1;
continue;
}
if input[i] == b'\\' {
escaped = true;
}
characters.push(input[i]);
i += 1;
}
if i == input.len() {
return (0, None);
}
return (i + 1, Some(Token::CharLit(character.into())));
return match characters.len() {
1 => {
if hex {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(number) = u8::from_str_radix(&string, 16) {
if number < 127 {
return (i, Some(Token::CharLit(number as char)));
}
}
}
}
(i, Some(Token::CharLit(characters[0] as char)))
}
2 => {
if hex {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(number) = u8::from_str_radix(&string, 16) {
if number < 127 {
return (i, Some(Token::CharLit(number as char)));
}
}
}
}
if unicode {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Some(character) = string.chars().next() {
return (i, Some(Token::CharLit(character)));
}
}
}
(0, None)
}
3 => {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if octal {
if let Ok(number) = u8::from_str_radix(&string, 8) {
if number < 127 {
return (i, Some(Token::CharLit(number as char)));
}
}
}
if let Some(character) = string.chars().next() {
return (i, Some(Token::CharLit(character)));
}
}
(0, None)
}
4 | 5 | 6 => {
if let Ok(unicode_string) = std::str::from_utf8(&characters[..]) {
if let Ok(hex) = u32::from_str_radix(&unicode_string, 16) {
if let Some(unicode_char) = std::char::from_u32(hex) {
return (i, Some(Token::CharLit(unicode_char)));
}
}
}
(0, None)
}
_ => (0, None),
};
// while i < input.len() {
// if !in_escape {
// if input[i] == b'\'' {
// break;
// }
// if input[i] == b'\\' {
// in_escape = !in_escape;
// } else {
// character.push(input[i] as char);
// }
// } else {
// in_escape = false;
// if input[i] == b'u' {
// i += 2;
// let mut j = i;
// let mut size = 0;
// while input[j] != b'}' {
// j += 1;
// size += 1;
// }
// let hex_string_number: String = input_tendril.subtendril(i as u32, size).to_string();
// if let Ok(hex) = u32::from_str_radix(&hex_string_number, 16) {
// if let Some(unicode) = std::char::from_u32(hex) {
// i = j;
// character = unicode.to_string();
// }
// } else {
// return (0, None);
// }
// } else {
// character.push(input[i] as char);
// }
// }
// i += 1;
// }
}
x if x.is_ascii_digit() => {
return Self::eat_integer(&input_tendril);

2
parser/src/tokenizer/token.rs
View File

@ -47,7 +47,7 @@ pub enum Token {
True,
False,
AddressLit(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
CharLit(#[serde(with = "leo_ast::common::tendril_json")] StrTendril),
CharLit(char),
At,

18
tests/parser/expression/literal/char.leo
View File

@ -4,8 +4,16 @@ expectation: Pass
*/
'a'
'A'
'\''
'\\'
'\n'
'\u{2764}'
'Z'
'\"'
'\t'
'\r'
'\0'
'\u{2764}'
'\u{306E}'
'❤'
'の'
'\xH2A'
'\xH9'
'\xO011'
'\xO172'

10
tests/parser/expression/literal/char_parse.leo
View File

@ -4,9 +4,15 @@ expectation: Pass
*/
'a'
'b'
'Z'
'\"'
'\t'
'\r'
'\0'
'\u{2764}'
'\u{2764}'
'\u{306E}'
'❤'
'\xH2A'
'\xH9'
'\xO011'
'\xO172'