diff --git a/parser/src/tokenizer/lexer.rs b/parser/src/tokenizer/lexer.rs index 0367ac0c87..b966d48591 100644 --- a/parser/src/tokenizer/lexer.rs +++ b/parser/src/tokenizer/lexer.rs @@ -120,7 +120,7 @@ impl Token { b'u' => { if let Some((start, open_brace)) = iter.next() { if *open_brace == b'{' { - let mut unicode: Vec = Vec::new(); + let mut characters: Vec = Vec::new(); while let Some((end, symbol)) = iter.next() { if end > start + 7 { @@ -128,12 +128,14 @@ impl Token { } match *symbol { - 0..=9 | b'a'..=b'f' | b'A'..=b'F' => unicode.push(*symbol), + 0..=9 | b'a'..=b'f' | b'A'..=b'F' => characters.push(*symbol), b'}' => { - if let Ok(string) = std::str::from_utf8(&unicode[..]) { - if let Some(character) = string.chars().next() { - collect.push(character); - break; + if let Ok(unicode_string) = std::str::from_utf8(&characters[..]) { + if let Ok(hex) = u32::from_str_radix(&unicode_string, 16) { + if let Some(unicode_char) = std::char::from_u32(hex) { + collect.push(unicode_char); + break; + } } } diff --git a/tests/expectations/parser/parser/expression/string.leo.out b/tests/expectations/parser/parser/expression/string.leo.out index 5240d494bc..70b43b4443 100644 --- a/tests/expectations/parser/parser/expression/string.leo.out +++ b/tests/expectations/parser/parser/expression/string.leo.out @@ -61,16 +61,16 @@ outputs: content: "`{ ] [ ; a`" - Value: String: - - - a + - - ࿺ - line_start: 1 line_stop: 1 col_start: 1 - col_stop: 13 + col_stop: 10 path: test - content: "`\\u{afafaf}`" + content: "`\\u{FFA}`" - Value: String: - - - a + - - 򯫺 - line_start: 1 line_stop: 1 col_start: 1 @@ -79,7 +79,7 @@ outputs: content: "`\\u{afafa}`" - Value: String: - - - a + - - 꾯 - line_start: 1 line_stop: 1 col_start: 1 @@ -88,7 +88,7 @@ outputs: content: "`\\u{afaf}`" - Value: String: - - - a + - - ૺ - line_start: 1 line_stop: 1 col_start: 1 @@ -97,7 +97,7 @@ outputs: content: "`\\u{afa}`" - Value: String: - - - a + - - ¯ - line_start: 1 line_stop: 1 col_start: 1 @@ -106,7 +106,7 @@ outputs: content: "`\\u{af}`" - Value: String: - - - a + - - "\n" - line_start: 1 line_stop: 1 col_start: 1 diff --git a/tests/parser/expression/string.leo b/tests/parser/expression/string.leo index 6af1a2ed34..47f75535df 100644 --- a/tests/parser/expression/string.leo +++ b/tests/parser/expression/string.leo @@ -9,7 +9,7 @@ expectation: Pass `{ ] [ ; a` -`\u{afafaf}` +`\u{FFA}` `\u{afafa}` `\u{afaf}` `\u{afa}`