From 1fa587400d5e55b71ca9632013370e0f95c177dc Mon Sep 17 00:00:00 2001 From: imaqtkatt Date: Thu, 11 Jan 2024 10:26:18 -0300 Subject: [PATCH] Support longer and shorter escape sequences --- src/term/parser/lexer.rs | 6 +++--- tests/golden_tests/run_file/world.hvm | 1 + tests/snapshots/run_file__world.hvm.snap | 5 +++++ 3 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tests/golden_tests/run_file/world.hvm create mode 100644 tests/snapshots/run_file__world.hvm.snap diff --git a/src/term/parser/lexer.rs b/src/term/parser/lexer.rs index bcd304de..1f74016b 100644 --- a/src/term/parser/lexer.rs +++ b/src/term/parser/lexer.rs @@ -36,7 +36,7 @@ pub enum Token { #[regex(r#""([^"\\]|\\t|\\u|\\n|\\")*""#, |lex| normalized_string(lex).ok())] Str(String), - #[regex(r#"'(.|\\t|\\u[0-9a-fA-F]{4}|\\n|\\')'"#, normalized_char)] + #[regex(r#"'(.|\\t|\\u[0-9a-fA-F]{1,8}|\\n|\\')'"#, normalized_char)] Char(u64), #[token("#")] @@ -145,7 +145,7 @@ fn normalized_string(lexer: &mut Lexer) -> Result Some('n') => s.push('\n'), Some('t') => s.push('\t'), Some('u') => { - let hex = chars.take(4).collect::(); + let hex = chars.take(8).collect::(); let hex_val = u32::from_str_radix(&hex, 16)?; let char = char::from_u32(hex_val).unwrap_or(char::REPLACEMENT_CHARACTER); s.push(char); @@ -219,7 +219,7 @@ fn normalized_char(lexer: &mut Lexer) -> Option { Some('t') => '\t', Some('\'') => '\'', Some('u') => { - let hex = chars.take(4).collect::(); + let hex = chars.take(8).collect::(); let hex_val = u32::from_str_radix(&hex, 16).unwrap(); char::from_u32(hex_val).unwrap_or(char::REPLACEMENT_CHARACTER) } diff --git a/tests/golden_tests/run_file/world.hvm b/tests/golden_tests/run_file/world.hvm new file mode 100644 index 00000000..b6584edd --- /dev/null +++ b/tests/golden_tests/run_file/world.hvm @@ -0,0 +1 @@ +main = (1, @#str s ('\u1F30E', s)) \ No newline at end of file diff --git a/tests/snapshots/run_file__world.hvm.snap b/tests/snapshots/run_file__world.hvm.snap new file mode 100644 index 00000000..456670d1 --- /dev/null +++ b/tests/snapshots/run_file__world.hvm.snap @@ -0,0 +1,5 @@ +--- +source: tests/golden_tests.rs +input_file: tests/golden_tests/run_file/world.hvm +--- +"🌎"