AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2024-12-01 10:17:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

wasn't capped at 10FFF for unicode

Browse Source
This commit is contained in:
gluax 2021-05-19 11:46:50 -04:00
parent edf236158d
commit 8cea284427
1 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
parser/src/tokenizer/lexer.rs
View File

@ -146,6 +146,19 @@ impl Token {
(i, Some(Token::CharLit(characters[0] as char)))
}
2 | 3 | 4 | 5 if unicode => {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(hex) = u32::from_str_radix(&string, 16) {
if hex <= 0x10FFF {
if let Some(unicode_char) = std::char::from_u32(hex) {
return (i, Some(Token::CharLit(unicode_char)));
}
}
}
}
(0, None)
}
2 => {
if hex {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
@ -157,14 +170,6 @@ impl Token {
}
}
if unicode {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Some(character) = string.chars().next() {
return (i, Some(Token::CharLit(character)));
}
}
}
(0, None)
}
3 => {