From 9ce7dab0ed556d7614a57faf29bab825961bcca7 Mon Sep 17 00:00:00 2001 From: Alessandro Coglio Date: Wed, 10 Jul 2024 08:59:27 -0700 Subject: [PATCH] Clarify ABNF vs. lexer/parser annotations. THis is only a documentation change. --- compiler/parser/src/parser/file.rs | 2 +- compiler/parser/src/tokenizer/token.rs | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/compiler/parser/src/parser/file.rs b/compiler/parser/src/parser/file.rs index 92e743991f..afc79846a9 100644 --- a/compiler/parser/src/parser/file.rs +++ b/compiler/parser/src/parser/file.rs @@ -327,7 +327,7 @@ impl ParserContext<'_, N> { let span = start + identifier.span; // TODO: Verify that this check is sound. - // Check that there is no whitespace in between the `@` symbol and identifier. + // Check that there is no whitespace or comments in between the `@` symbol and identifier. match identifier.span.hi.0 - start.lo.0 > 1 + identifier.name.to_string().len() as u32 { true => Err(ParserError::space_in_annotation(span).into()), false => Ok(Annotation { identifier, span, id: self.node_builder.next_id() }), diff --git a/compiler/parser/src/tokenizer/token.rs b/compiler/parser/src/tokenizer/token.rs index f810cea96c..798753ff5a 100644 --- a/compiler/parser/src/tokenizer/token.rs +++ b/compiler/parser/src/tokenizer/token.rs @@ -102,10 +102,18 @@ pub enum Token { Arrow, BigArrow, Underscore, - At, + At, // @ is not a symbol token in the ABNF grammar (see explanation about annotations below) // There is no symbol for `)group` here (unlike the ABNF grammar), // because we handle that differently in the lexer. + // The ABNF grammar has annotations as tokens, + // defined as @ immediately followed by an identifier. + // Here instead we regard the @ sign alone as a token (see `At` above), + // and we lex it separately from the identifier that is supposed to follow it in an annotation. + // When parsing annotations, we check that there is no whitespace or comments + // between the @ and the identifier, thus eventually complying to the ABNF grammar. + // See the parse_annotation function. + // Type keywords Address, Bool,