_ token in parser and symbol in abnf

grammar/abnf-grammar.txt

@@ -645,7 +645,7 @@ symbol = "!" / "&&" / "||"
        / "[" / "]"
        / "{" / "}"
        / "," / "." / ".." / "..." / ";" / ":" / "::" / "?"
-       / "->"
+       / "->" / "_"
 
 ; Everything defined above, other than comments and whitespace,
 ; is a token, as defined by the following rule.

parser/src/parser/context.rs

@@ -153,7 +153,7 @@ impl ParserContext {
                 }
                 _ => GroupCoordinate::SignLow,
             },
-            Token::Ident(x) if x == "_" => GroupCoordinate::Inferred,
+            Token::Underscore => GroupCoordinate::Inferred,
             Token::Int(value) => GroupCoordinate::Number(value.clone(), token.span.clone()),
             _ => return None,
         })

parser/src/tokenizer/lexer.rs

@@ -43,14 +43,11 @@ fn eat_identifier(input: &[u8]) -> Option<(&[u8], &[u8])> {
     if input.is_empty() {
         return None;
     }
-    if !input[0].is_ascii_alphabetic() && input[0] != b'_' {
+    if !input[0].is_ascii_alphabetic() {
         // Allow _ at start.
         return None;
     }
-    if input.len() == 1 && input[0] == b'_' {
-        // But only if it the length of the identifer is 1.
-        return None;
-    }
+
     let mut i = 1usize;
     while i < input.len() {
         if !input[i].is_ascii_alphanumeric() && input[i] != b'_' {
@@ -283,6 +280,7 @@ impl Token {
                 }
                 return (&input[1..], Some(Token::Assign));
             }
+            b'_' => return (&input[1..], Some(Token::Underscore)),
             b'@' => return (&input[1..], Some(Token::At)),
             b'[' => return (&input[1..], Some(Token::LeftSquare)),
             b']' => return (&input[1..], Some(Token::RightSquare)),

parser/src/tokenizer/mod.rs

@@ -172,6 +172,7 @@ mod tests {
         -
         -=
         ->
+        _
         .
         ..
         ...
@@ -221,7 +222,7 @@ mod tests {
         }
         assert_eq!(
             output,
-            r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || & &= | |= ^ ^= ~ << <<= >> >>= >>> >>>= % %= ||= &&= ? // test
+            r#""test" "test{}test" "test{}" "{}test" "test{" "test}" "test{test" "test}test" "te{{}}" aleo1qnr4dkkvkgfqph0vzc3y6z2eu975wnpz2925ntjccd5cfqxtyu8sta57j8 test_ident 12345 address as bool circuit const else false field for function group i128 i64 i32 i16 i8 if import in input let mut return static string test true u128 u64 u32 u16 u8 self Self console ! != && ( ) * ** **= *= + += , - -= -> _ . .. ... / /= : :: ; < <= = == > >= @ [ ] { { } } || & &= | |= ^ ^= ~ << <<= >> >>= >>> >>>= % %= ||= &&= ? // test
  /* test */ //
  "#
         );

parser/src/tokenizer/token.rs

@@ -85,6 +85,7 @@ pub enum Token {
     DoubleColon,
     Question,
     Arrow,
+    Underscore,
 
     // Syntactic Grammr
     // Types
@@ -298,6 +299,7 @@ impl fmt::Display for Token {
             OrEq => write!(f, "||="),
             AndEq => write!(f, "&&="),
             Question => write!(f, "?"),
+            Underscore => write!(f, "_"),
         }
     }
 }