ProvableHQ/leo
1
1
Fork 0
mirror of https://github.com/ProvableHQ/leo.git synced 2024-09-21 02:37:47 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix unicode chars

Browse Source
This commit is contained in:
damirka 2021-05-19 23:59:54 +03:00
parent 737985034b
commit ed03a385b5
3 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
parser/src/tokenizer/lexer.rs
View File

@ -120,7 +120,7 @@ impl Token {
b'u' => {
if let Some((start, open_brace)) = iter.next() {
if *open_brace == b'{' {
let mut unicode: Vec<u8> = Vec::new();
let mut characters: Vec<u8> = Vec::new();
while let Some((end, symbol)) = iter.next() {
if end > start + 7 {
@ -128,12 +128,14 @@ impl Token {
}
match *symbol {
0..=9 | b'a'..=b'f' | b'A'..=b'F' => unicode.push(*symbol),
0..=9 | b'a'..=b'f' | b'A'..=b'F' => characters.push(*symbol),
b'}' => {
if let Ok(string) = std::str::from_utf8(&unicode[..]) {
if let Some(character) = string.chars().next() {
collect.push(character);
break;
if let Ok(unicode_string) = std::str::from_utf8(&characters[..]) {
if let Ok(hex) = u32::from_str_radix(&unicode_string, 16) {
if let Some(unicode_char) = std::char::from_u32(hex) {
collect.push(unicode_char);
break;
}
}
}

16
tests/expectations/parser/parser/expression/string.leo.out
View File

@ -61,16 +61,16 @@ outputs:
content: "`{ ] [ ; a`"
- Value:
String:
- - a
- - ࿺
- line_start: 1
line_stop: 1
col_start: 1
col_stop: 13
col_stop: 10
path: test
content: "`\\u{afafaf}`"
content: "`\\u{FFA}`"
- Value:
String:
- - a
- - 򯫺
- line_start: 1
line_stop: 1
col_start: 1
@ -79,7 +79,7 @@ outputs:
content: "`\\u{afafa}`"
- Value:
String:
- - a
- - 꾯
- line_start: 1
line_stop: 1
col_start: 1
@ -88,7 +88,7 @@ outputs:
content: "`\\u{afaf}`"
- Value:
String:
- - a
- - ૺ
- line_start: 1
line_stop: 1
col_start: 1
@ -97,7 +97,7 @@ outputs:
content: "`\\u{afa}`"
- Value:
String:
- - a
- - ¯
- line_start: 1
line_stop: 1
col_start: 1
@ -106,7 +106,7 @@ outputs:
content: "`\\u{af}`"
- Value:
String:
- - a
- - "\n"
- line_start: 1
line_stop: 1
col_start: 1

2
tests/parser/expression/string.leo
View File

@ -9,7 +9,7 @@ expectation: Pass
`{ ] [ ; a`
`\u{afafaf}`
`\u{FFA}`
`\u{afafa}`
`\u{afaf}`
`\u{afa}`