AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2024-11-28 19:23:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

clean up lexing char

Browse Source
This commit is contained in:
gluax 2021-05-19 12:09:02 -04:00
parent 8cea284427
commit 6f8df3c84d
1 changed files with 16 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
parser/src/tokenizer/lexer.rs
View File

@ -78,8 +78,10 @@ impl Token {
let mut characters: Vec<u8> = vec![];
while i < input.len() {
println!("?? input[i] {}", input[i]);
if !escaped {
if input[i] == b'\'' {
println!("last");
last = true;
i += 1;
break;
@ -139,14 +141,7 @@ impl Token {
}
return match characters.len() {
1 => {
if hex {
return (0, None);
}
(i, Some(Token::CharLit(characters[0] as char)))
}
2 | 3 | 4 | 5 if unicode => {
1 | 2 | 3 | 4 | 5 if unicode => {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(hex) = u32::from_str_radix(&string, 16) {
if hex <= 0x10FFF {
@ -159,20 +154,26 @@ impl Token {
(0, None)
}
2 => {
1 => {
if hex {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(number) = u8::from_str_radix(&string, 16) {
if number <= 127 {
return (i, Some(Token::CharLit(number as char)));
}
return (0, None);
}
(i, Some(Token::CharLit(characters[0] as char)))
}
2 if hex => {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(number) = u8::from_str_radix(&string, 16) {
if number <= 127 {
return (i, Some(Token::CharLit(number as char)));
}
}
}
(0, None)
}
3 => {
3 | 4 => {
// direct unicode symbol
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Some(character) = string.chars().next() {
return (i, Some(Token::CharLit(character)));
@ -181,17 +182,6 @@ impl Token {
(0, None)
}
4 | 5 | 6 => {
if let Ok(unicode_string) = std::str::from_utf8(&characters[..]) {
if let Ok(hex) = u32::from_str_radix(&unicode_string, 16) {
if let Some(unicode_char) = std::char::from_u32(hex) {
return (i, Some(Token::CharLit(unicode_char)));
}
}
}
(0, None)
}
_ => (0, None),
};
}