diff --git a/crates/kind-cli/Cargo.toml b/crates/kind-cli/Cargo.toml index 56d6e0f7..3e65626a 100644 --- a/crates/kind-cli/Cargo.toml +++ b/crates/kind-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kind-cli" -version = "0.3.2" +version = "0.3.3" edition = "2021" description = "A pure functional functional language that uses the HVM." repository = "https://github.com/Kindelia/Kind2" diff --git a/crates/kind-parser/src/lexer/comments.rs b/crates/kind-parser/src/lexer/comments.rs index 59ed64c4..62025163 100644 --- a/crates/kind-parser/src/lexer/comments.rs +++ b/crates/kind-parser/src/lexer/comments.rs @@ -11,11 +11,13 @@ impl<'a> Lexer<'a> { /// Single line comments pub fn lex_comment(&mut self, start: usize) -> (Token, Range) { self.next_char(); + let mut is_doc = false; if let Some('/') = self.peekable.peek() { self.next_char(); is_doc = true; } + let cmt = self.accumulate_while(&|x| x != '\n'); ( Token::Comment(is_doc, cmt.to_string()), diff --git a/crates/kind-parser/src/lexer/mod.rs b/crates/kind-parser/src/lexer/mod.rs index ad988c27..34493e70 100644 --- a/crates/kind-parser/src/lexer/mod.rs +++ b/crates/kind-parser/src/lexer/mod.rs @@ -57,8 +57,9 @@ impl<'a> Lexer<'a> { } } - pub fn get_next_no_error(&mut self, vec: Sender>) -> (Token, Range) { + pub fn get_next_no_error(&mut self, vec: Sender>) -> (bool, Token, Range) { loop { + let is_break = self.is_linebreak(); let (token, span) = self.lex_token(); match token { Token::Error(x) => { @@ -68,11 +69,11 @@ impl<'a> Lexer<'a> { Token::Comment(false, _) => continue, _ => (), } - return (token, span); + return (is_break, token, span); } } - pub fn lex_token(&mut self) -> (Token, Range) { + fn lex_token(&mut self) -> (Token, Range) { let start = self.span(); match self.peekable.peek() { None => (Token::Eof, self.mk_range(start)), diff --git a/crates/kind-parser/src/lexer/tokens.rs b/crates/kind-parser/src/lexer/tokens.rs index a8e7bd47..177d1d4e 100644 --- a/crates/kind-parser/src/lexer/tokens.rs +++ b/crates/kind-parser/src/lexer/tokens.rs @@ -169,7 +169,8 @@ impl fmt::Display for Token { Token::Bang => write!(f, "!"), Token::HashHash => write!(f, "##"), Token::Hash => write!(f, "#"), - Token::Comment(_, _) => write!(f, "docstring comment"), + Token::Comment(true, comment) => write!(f, "docstring '{comment}'"), + Token::Comment(false, comment) => write!(f, "comment '{comment}'"), Token::Eof => write!(f, "End of file"), Token::Error(_) => write!(f, "ERROR"), Token::Return => write!(f, "return"), diff --git a/crates/kind-parser/src/state.rs b/crates/kind-parser/src/state.rs index 1c212fab..02832fb0 100644 --- a/crates/kind-parser/src/state.rs +++ b/crates/kind-parser/src/state.rs @@ -33,9 +33,12 @@ impl<'a> Parser<'a> { pub fn new(mut lexer: Lexer<'a>, sender: Sender>) -> Parser<'a> { let mut queue = VecDeque::with_capacity(3); let mut breaks = VecDeque::with_capacity(3); + for _ in 0..3 { - breaks.push_back(lexer.is_linebreak()); - queue.push_back(lexer.get_next_no_error(sender.clone())); + let (is_break, token, range) = lexer.get_next_no_error(sender.clone()); + + breaks.push_back(is_break); + queue.push_back((token, range)); } Parser { lexer, @@ -50,9 +53,12 @@ impl<'a> Parser<'a> { pub fn advance(&mut self) -> (Token, Range) { let cur = self.queue.pop_front().unwrap(); self.breaks.pop_front(); - self.breaks.push_back(self.lexer.is_linebreak()); - self.queue - .push_back(self.lexer.get_next_no_error(self.dignostic_channel.clone())); + + let (is_break, token, range) = self.lexer.get_next_no_error(self.dignostic_channel.clone()); + + self.breaks.push_back(is_break); + self.queue.push_back((token, range)); + self.eaten += 1; cur }