From c8b44141ebb44ac99d55171688964973b9934e22 Mon Sep 17 00:00:00 2001 From: gluax <16431709+gluax@users.noreply.github.com> Date: Mon, 18 Apr 2022 11:33:43 -0700 Subject: [PATCH] fix some span and input parsing issues --- .../ast/src/reducer/reconstructing_reducer.rs | 2 +- compiler/parser/src/parser/context.rs | 2 +- compiler/parser/src/parser/file.rs | 4 +-- compiler/parser/src/parser/input.rs | 2 +- compiler/parser/src/parser/statement.rs | 4 +-- compiler/parser/src/parser/type_.rs | 32 +++++++++++-------- compiler/parser/src/tokenizer/mod.rs | 4 +-- 7 files changed, 27 insertions(+), 23 deletions(-) diff --git a/compiler/ast/src/reducer/reconstructing_reducer.rs b/compiler/ast/src/reducer/reconstructing_reducer.rs index 9d0bc2e46d..308fb36ec7 100644 --- a/compiler/ast/src/reducer/reconstructing_reducer.rs +++ b/compiler/ast/src/reducer/reconstructing_reducer.rs @@ -154,7 +154,7 @@ pub trait ReconstructingReducer { value: Expression, ) -> Result { Ok(DefinitionStatement { - declaration_type: definition.declaration_type.clone(), + declaration_type: definition.declaration_type, variable_names, type_, value, diff --git a/compiler/parser/src/parser/context.rs b/compiler/parser/src/parser/context.rs index 1b82207c1a..e4123a94a5 100644 --- a/compiler/parser/src/parser/context.rs +++ b/compiler/parser/src/parser/context.rs @@ -114,7 +114,7 @@ impl<'a> ParserContext<'a> { Some(idx) => idx, }; - looker(self.tokens.get(idx).unwrap_or_else(|| &self.dummy_eof)) + looker(self.tokens.get(idx).unwrap_or(&self.dummy_eof)) } /// Emit the error `err`. diff --git a/compiler/parser/src/parser/file.rs b/compiler/parser/src/parser/file.rs index d925079b9b..bcf9104534 100644 --- a/compiler/parser/src/parser/file.rs +++ b/compiler/parser/src/parser/file.rs @@ -100,7 +100,7 @@ impl ParserContext<'_> { } self.expect(&Token::Colon)?; - let type_ = self.parse_type()?.0; + let type_ = self.parse_all_types()?.0; Ok(FunctionInput::Variable(FunctionInputVariable::new( name.clone(), mode, @@ -124,7 +124,7 @@ impl ParserContext<'_> { // Parse return type. let output = if self.eat(&Token::Arrow) { - Some(self.parse_type()?.0) + Some(self.parse_all_types()?.0) } else { None }; diff --git a/compiler/parser/src/parser/input.rs b/compiler/parser/src/parser/input.rs index 16252416b4..05dba6eda1 100644 --- a/compiler/parser/src/parser/input.rs +++ b/compiler/parser/src/parser/input.rs @@ -65,7 +65,7 @@ impl ParserContext<'_> { let name = self.expect_ident()?; self.expect(&Token::Colon)?; - let (type_, span) = self.parse_type()?; + let (type_, span) = self.parse_non_ident_types()?; self.expect(&Token::Assign)?; let value = self.parse_primary_expression()?; self.expect(&Token::Semicolon)?; diff --git a/compiler/parser/src/parser/statement.rs b/compiler/parser/src/parser/statement.rs index bfdc238a2e..df33528786 100644 --- a/compiler/parser/src/parser/statement.rs +++ b/compiler/parser/src/parser/statement.rs @@ -252,8 +252,6 @@ impl ParserContext<'_> { Token::Const => Declare::Const, _ => unreachable!("parse_definition_statement_ shouldn't produce this"), }; - dbg!(); - // Parse variable names. let variable_names = if self.peek_is_left_par() { let vars = self @@ -272,7 +270,7 @@ impl ParserContext<'_> { // Parse an optional type ascription. let type_ = self .eat(&Token::Colon) - .then(|| self.parse_type().map(|t| t.0)) + .then(|| self.parse_all_types().map(|t| t.0)) .transpose()?; self.expect(&Token::Assign)?; diff --git a/compiler/parser/src/parser/type_.rs b/compiler/parser/src/parser/type_.rs index 7d4fc5c0dc..fc560e9b4e 100644 --- a/compiler/parser/src/parser/type_.rs +++ b/compiler/parser/src/parser/type_.rs @@ -57,23 +57,29 @@ impl ParserContext<'_> { /// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type. /// Also returns the span of the parsed token. - pub fn parse_type(&mut self) -> Result<(Type, Span)> { + pub fn parse_non_ident_types(&mut self) -> Result<(Type, Span)> { + let span = self.expect_any(TYPE_TOKENS)?; + Ok(( + match &self.prev_token.token { + Token::Field => Type::Field, + Token::Group => Type::Group, + Token::Address => Type::Address, + Token::Bool => Type::Boolean, + Token::Char => Type::Char, + x => Type::IntegerType(Self::token_to_int_type(x).expect("invalid int type")), + }, + span, + )) + } + + /// Returns a [`(Type, Span)`] tuple of AST nodes if the next token represents a type. + /// Also returns the span of the parsed token. + pub fn parse_all_types(&mut self) -> Result<(Type, Span)> { Ok(if let Some(ident) = self.eat_identifier() { let span = ident.span.clone(); (Type::Identifier(ident), span) } else { - let span = self.expect_any(TYPE_TOKENS)?; - ( - match &self.prev_token.token { - Token::Field => Type::Field, - Token::Group => Type::Group, - Token::Address => Type::Address, - Token::Bool => Type::Boolean, - Token::Char => Type::Char, - x => Type::IntegerType(Self::token_to_int_type(x).expect("invalid int type")), - }, - span, - ) + self.parse_non_ident_types()? }) } } diff --git a/compiler/parser/src/tokenizer/mod.rs b/compiler/parser/src/tokenizer/mod.rs index 0d97f13ea3..4829268689 100644 --- a/compiler/parser/src/tokenizer/mod.rs +++ b/compiler/parser/src/tokenizer/mod.rs @@ -46,7 +46,7 @@ pub(crate) fn tokenize_iter<'a>(path: &'a str, input: &'a str) -> impl 'a + Iter iter::from_fn(move || { while input.len() > index { let token = match Token::eat(&input[index..]) { - Err(e) => return Some(Err(e.into())), + Err(e) => return Some(Err(e)), Ok(t) => t, }; @@ -56,7 +56,7 @@ pub(crate) fn tokenize_iter<'a>(path: &'a str, input: &'a str) -> impl 'a + Iter if bytes[index] == 0x000D && matches!(bytes.get(index + 1), Some(0x000A)) { // Check carriage return followed by newline. line_no += 1; - line_start = index + token_len; + line_start = index + token_len + 1; index += token_len; } else if matches!(bytes[index], 0x000A | 0x000D) { // Check new-line or carriage-return