AleoHQ/leo
1
1
Fork 0
mirror of https://github.com/AleoHQ/leo.git synced 2024-11-29 03:35:10 +03:00
Code Issues Packages Projects Releases Wiki Activity

wasn't capped at 10FFF for unicode

Browse Source
This commit is contained in:
gluax 2021-05-19 11:46:50 -04:00
parent edf236158d
commit 8cea284427
1 changed files with 13 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
parser/src/tokenizer/lexer.rs
View File

@ -146,6 +146,19 @@ impl Token {
(i, Some(Token::CharLit(characters[0] as char))) (i, Some(Token::CharLit(characters[0] as char)))
} }
2 | 3 | 4 | 5 if unicode => {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Ok(hex) = u32::from_str_radix(&string, 16) {
if hex <= 0x10FFF {
if let Some(unicode_char) = std::char::from_u32(hex) {
return (i, Some(Token::CharLit(unicode_char)));
}
}
}
}
(0, None)
}
2 => { 2 => {
if hex { if hex {
if let Ok(string) = std::str::from_utf8(&characters[..]) { if let Ok(string) = std::str::from_utf8(&characters[..]) {
@ -157,14 +170,6 @@ impl Token {
} }
} }
if unicode {
if let Ok(string) = std::str::from_utf8(&characters[..]) {
if let Some(character) = string.chars().next() {
return (i, Some(Token::CharLit(character)));
}
}
}
(0, None) (0, None)
} }
3 => { 3 => {