From 0435b7ce831238034b35cad6ebd6c426ac962884 Mon Sep 17 00:00:00 2001 From: gluax Date: Fri, 14 May 2021 13:07:54 -0400 Subject: [PATCH] remove octal, fix ascii hex --- compiler/tests/char/circuit.leo | 5 +-- compiler/tests/char/hex.leo | 4 +-- compiler/tests/char/input/char.in | 3 +- compiler/tests/char/mod.rs | 8 ----- compiler/tests/char/octal.leo | 4 --- compiler/tests/char/unicode.leo | 3 ++ input/src/leo-input.pest | 4 +-- input/src/values/char_types.rs | 20 ------------ parser/src/tokenizer/lexer.rs | 32 ++----------------- .../parser/expression/literal/char.leo.out | 5 +-- .../expression/literal/char_fail.leo.out | 1 + .../expression/literal/char_parse.leo.out | 29 +---------------- tests/parser/expression/literal/char.leo | 5 +-- tests/parser/expression/literal/char_fail.leo | 2 ++ .../parser/expression/literal/char_parse.leo | 5 +-- 15 files changed, 21 insertions(+), 109 deletions(-) delete mode 100644 compiler/tests/char/octal.leo diff --git a/compiler/tests/char/circuit.leo b/compiler/tests/char/circuit.leo index 1a287668ba..1a39fb704a 100644 --- a/compiler/tests/char/circuit.leo +++ b/compiler/tests/char/circuit.leo @@ -2,8 +2,9 @@ circuit Foo { a: char; b: char; c: char; + d: char; } -function main(a: char, b: char, c: char) { - let f = Foo { a, b, c }; +function main(a: char, b: char, c: char, d: char) { + let f = Foo { a, b, c, d }; } \ No newline at end of file diff --git a/compiler/tests/char/hex.leo b/compiler/tests/char/hex.leo index 2cbc144783..e3d67ac5ea 100644 --- a/compiler/tests/char/hex.leo +++ b/compiler/tests/char/hex.leo @@ -1,4 +1,4 @@ function main() { - const heart: char = '❤'; - const Hiragana = 'の'; + const star = '\x2A'; + const star = '\x7F'; } \ No newline at end of file diff --git a/compiler/tests/char/input/char.in b/compiler/tests/char/input/char.in index 458c17adc6..872372a034 100644 --- a/compiler/tests/char/input/char.in +++ b/compiler/tests/char/input/char.in @@ -1,4 +1,5 @@ [main] a: char = 'a'; b: char = '\''; -c: char = '\u{2764}'; \ No newline at end of file +c: char = '\u{2764}'; +d: char = '\x2A'; \ No newline at end of file diff --git a/compiler/tests/char/mod.rs b/compiler/tests/char/mod.rs index 21e872ef45..51f9171c92 100644 --- a/compiler/tests/char/mod.rs +++ b/compiler/tests/char/mod.rs @@ -76,14 +76,6 @@ fn test_function() { assert_satisfied(program); } -#[test] -fn test_octal() { - let program_string = include_str!("octal.leo"); - let program = parse_program(program_string).unwrap(); - - assert_satisfied(program); -} - #[test] fn test_unicode() { let program_string = include_str!("unicode.leo"); diff --git a/compiler/tests/char/octal.leo b/compiler/tests/char/octal.leo deleted file mode 100644 index 8591646ba9..0000000000 --- a/compiler/tests/char/octal.leo +++ /dev/null @@ -1,4 +0,0 @@ -function main() { - const tab: char = '\xO011'; - const z = '\xO172'; -} \ No newline at end of file diff --git a/compiler/tests/char/unicode.leo b/compiler/tests/char/unicode.leo index 0cd61be0f5..39d88d782d 100644 --- a/compiler/tests/char/unicode.leo +++ b/compiler/tests/char/unicode.leo @@ -1,4 +1,7 @@ function main() { const heart: char = '\u{2764}'; const Hiragana = '\u{306E}'; + + const heart2: char = '❤'; + const Hiragana2 = 'の'; } \ No newline at end of file diff --git a/input/src/leo-input.pest b/input/src/leo-input.pest index 99874fd882..a500de21fb 100644 --- a/input/src/leo-input.pest +++ b/input/src/leo-input.pest @@ -137,15 +137,13 @@ number_positive = @{ ASCII_DIGIT+ } // ANY is equivalent to '\u{00}'..'\u{10FFFF}' basic_char = { ANY } escaped_char = @{ "\\" ~ ("\"" | "\'" | "\\" | "/" | "b" | "f" | "n" | "r" | "t") } -hex_char = @{ "\\" ~ "x" ~ "H" ~ ASCII_HEX_DIGIT{1, 2} } -octal_char = @{ "\\" ~ "x" ~ "O" ~ ASCII_DIGIT{3} } +hex_char = @{ "\\" ~ "x" ~ ASCII_HEX_DIGIT{2} } unicode_char = @{ "\\" ~ "u" ~ "{" ~ ASCII_HEX_DIGIT{1, 6} ~ "}" } char_types = { escaped_char | unicode_char | hex_char - | octal_char | basic_char } diff --git a/input/src/values/char_types.rs b/input/src/values/char_types.rs index 5c0a56f3a5..314f4f5c9d 100644 --- a/input/src/values/char_types.rs +++ b/input/src/values/char_types.rs @@ -49,15 +49,6 @@ pub struct HexChar<'ast> { pub span: Span<'ast>, } -#[derive(Clone, Debug, FromPest, PartialEq, Eq)] -#[pest_ast(rule(Rule::octal_char))] -pub struct OctalChar<'ast> { - #[pest_ast(outer(with(span_into_string)))] - pub value: String, - #[pest_ast(outer())] - pub span: Span<'ast>, -} - #[derive(Clone, Debug, FromPest, PartialEq, Eq)] #[pest_ast(rule(Rule::unicode_char))] pub struct UnicodeChar<'ast> { @@ -73,7 +64,6 @@ pub enum CharTypes<'ast> { Basic(BasicChar<'ast>), Escaped(EscapedChar<'ast>), Hex(HexChar<'ast>), - Octal(OctalChar<'ast>), Unicode(UnicodeChar<'ast>), } @@ -104,16 +94,6 @@ impl<'ast> CharTypes<'ast> { Err(InputParserError::invalid_char(character.value, &character.span)) } - Self::Octal(character) => { - let octal_string_number = character.value[3..character.value.len()].to_string(); - if let Ok(number) = u8::from_str_radix(&octal_string_number, 8) { - if number < 127 { - return Ok(number as char); - } - } - - Err(InputParserError::invalid_char(character.value, &character.span)) - } Self::Unicode(character) => { let unicode_string_number = character.value[3..=character.value.len() - 2].to_string(); if let Ok(hex) = u32::from_str_radix(&unicode_string_number, 16) { diff --git a/parser/src/tokenizer/lexer.rs b/parser/src/tokenizer/lexer.rs index e55ac78e08..9984f9dab2 100644 --- a/parser/src/tokenizer/lexer.rs +++ b/parser/src/tokenizer/lexer.rs @@ -73,7 +73,6 @@ impl Token { let mut i = 1; let mut escaped = false; let mut hex = false; - let mut octal = false; let mut unicode = false; let mut last = false; let mut characters: Vec = vec![]; @@ -109,18 +108,7 @@ impl Token { b'\'' => characters.push(39), b'\\' => characters.push(92), b'x' => { - i += 1; - match input[i] { - b'H' => { - hex = true; - } - b'O' => { - octal = true; - } - _ => { - return (0, None); - } - } + hex = true; i += 1; continue; @@ -153,13 +141,7 @@ impl Token { return match characters.len() { 1 => { if hex { - if let Ok(string) = std::str::from_utf8(&characters[..]) { - if let Ok(number) = u8::from_str_radix(&string, 16) { - if number < 127 { - return (i, Some(Token::CharLit(number as char))); - } - } - } + return (0, None); } (i, Some(Token::CharLit(characters[0] as char))) @@ -168,7 +150,7 @@ impl Token { if hex { if let Ok(string) = std::str::from_utf8(&characters[..]) { if let Ok(number) = u8::from_str_radix(&string, 16) { - if number < 127 { + if number <= 127 { return (i, Some(Token::CharLit(number as char))); } } @@ -187,14 +169,6 @@ impl Token { } 3 => { if let Ok(string) = std::str::from_utf8(&characters[..]) { - if octal { - if let Ok(number) = u8::from_str_radix(&string, 8) { - if number < 127 { - return (i, Some(Token::CharLit(number as char))); - } - } - } - if let Some(character) = string.chars().next() { return (i, Some(Token::CharLit(character))); } diff --git a/tests/expectations/parser/parser/expression/literal/char.leo.out b/tests/expectations/parser/parser/expression/literal/char.leo.out index 0370692c2e..d6e0fd1fc1 100644 --- a/tests/expectations/parser/parser/expression/literal/char.leo.out +++ b/tests/expectations/parser/parser/expression/literal/char.leo.out @@ -12,7 +12,4 @@ outputs: - "'の' @ 1:1-11" - "'❤' @ 1:1-6" - "'の' @ 1:1-6" - - "'*' @ 1:1-8" - - "'' @ 1:1-7" - - "'' @ 1:1-9" - - "'z' @ 1:1-9" + - "'*' @ 1:1-7" diff --git a/tests/expectations/parser/parser/expression/literal/char_fail.leo.out b/tests/expectations/parser/parser/expression/literal/char_fail.leo.out index 0becec81b8..fb3cc4bdcc 100644 --- a/tests/expectations/parser/parser/expression/literal/char_fail.leo.out +++ b/tests/expectations/parser/parser/expression/literal/char_fail.leo.out @@ -5,3 +5,4 @@ outputs: - " --> test:1:1\n |\n 1 | '\\'\n | ^\n |\n = unexpected token: '''" - " --> test:1:1\n |\n 1 | 'a\n | ^\n |\n = unexpected token: '''" - " --> test:1:1\n |\n 1 | ''\n | ^\n |\n = unexpected token: '''" + - " --> test:1:1\n |\n 1 | '\\x9'\n | ^\n |\n = unexpected token: '''" diff --git a/tests/expectations/parser/parser/expression/literal/char_parse.leo.out b/tests/expectations/parser/parser/expression/literal/char_parse.leo.out index bcbbb81ddd..3b080cd0a6 100644 --- a/tests/expectations/parser/parser/expression/literal/char_parse.leo.out +++ b/tests/expectations/parser/parser/expression/literal/char_parse.leo.out @@ -95,36 +95,9 @@ outputs: - Value: Char: - "*" - - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 8 - path: test - content: "'\\xH2A'" - - Value: - Char: - - "\t" - line_start: 1 line_stop: 1 col_start: 1 col_stop: 7 path: test - content: "'\\xH9'" - - Value: - Char: - - "\t" - - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 9 - path: test - content: "'\\xO011'" - - Value: - Char: - - z - - line_start: 1 - line_stop: 1 - col_start: 1 - col_stop: 9 - path: test - content: "'\\xO172'" + content: "'\\x2A'" diff --git a/tests/parser/expression/literal/char.leo b/tests/parser/expression/literal/char.leo index 013de7f106..5ea47f7dbf 100644 --- a/tests/parser/expression/literal/char.leo +++ b/tests/parser/expression/literal/char.leo @@ -13,7 +13,4 @@ expectation: Pass '\u{306E}' '❤' 'の' -'\xH2A' -'\xH9' -'\xO011' -'\xO172' \ No newline at end of file +'\x2A' \ No newline at end of file diff --git a/tests/parser/expression/literal/char_fail.leo b/tests/parser/expression/literal/char_fail.leo index 3a4b06903e..565c6f3922 100644 --- a/tests/parser/expression/literal/char_fail.leo +++ b/tests/parser/expression/literal/char_fail.leo @@ -8,3 +8,5 @@ expectation: Fail 'a '' + +'\x9' \ No newline at end of file diff --git a/tests/parser/expression/literal/char_parse.leo b/tests/parser/expression/literal/char_parse.leo index 1bb25c489e..515f6b10f3 100644 --- a/tests/parser/expression/literal/char_parse.leo +++ b/tests/parser/expression/literal/char_parse.leo @@ -13,7 +13,4 @@ expectation: Pass '\u{306E}' '❤' 'の' -'\xH2A' -'\xH9' -'\xO011' -'\xO172' \ No newline at end of file +'\x2A' \ No newline at end of file