fix spans for comments

parser/src/tokenizer/lexer.rs

@@ -16,6 +16,7 @@
 
 use crate::tokenizer::{FormattedStringPart, Token};
 use leo_ast::Span;
+use serde::{Deserialize, Serialize};
 
 use std::fmt;
 
@@ -192,7 +193,7 @@ impl Token {
                 return (&input[1..], Some(Token::Dot));
             }
             b'/' => {
-                if let Some(input) = eat(input, "//") {
+                if eat(input, "//").is_some() {
                     let eol = input.iter().position(|x| *x == b'\n');
                     let (input, comment) = if let Some(eol) = eol {
                         (&input[(eol + 1)..], &input[..eol])
@@ -203,13 +204,13 @@ impl Token {
                         input,
                         Some(Token::CommentLine(String::from_utf8_lossy(comment).to_string())),
                     );
-                } else if let Some(input) = eat(input, "/*") {
+                } else if eat(input, "/*").is_some() {
                     if input.is_empty() {
                         return (input, None);
                     }
-                    let eol = input.windows(2).position(|x| x[0] == b'*' && x[1] == b'/');
+                    let eol = input.windows(2).skip(2).position(|x| x[0] == b'*' && x[1] == b'/');
                     let (input, comment) = if let Some(eol) = eol {
-                        (&input[(eol + 2)..], &input[..eol])
+                        (&input[(eol + 4)..], &input[..eol + 4])
                     } else {
                         (&input[input.len()..input.len()], &input[..])
                     };
@@ -347,7 +348,7 @@ impl Token {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct SpannedToken {
     pub token: Token,
     pub span: Span,

parser/src/tokenizer/mod.rs

@@ -41,17 +41,34 @@ pub(crate) fn tokenize(path: &str, source: &str) -> Result<Vec<SpannedToken>, To
     while !input.is_empty() {
         match Token::gobble(input) {
             (output, Some(token)) => {
-                let span = Span {
+                let mut span = Span {
                     line_start: line_no,
                     line_stop: line_no,
                     col_start: index - line_start + 1,
                     col_stop: index - line_start + (input.len() - output.len()) + 1,
                     path: path.clone(),
                 };
-                if let Token::AddressLit(address) = &token {
-                    if !validate_address(address) {
-                        return Err(TokenError::invalid_address_lit(address, &span));
+                match &token {
+                    Token::CommentLine(_) => {
+                        line_no += 1;
+                        line_start = index + (input.len() - output.len());
                     }
+                    Token::CommentBlock(block) => {
+                        let line_ct = block.chars().filter(|x| *x == '\n').count();
+                        line_no += line_ct;
+                        if line_ct > 0 {
+                            let last_line_index = block.rfind('\n').unwrap();
+                            line_start = index + last_line_index + 1;
+                            span.col_stop = index + (input.len() - output.len()) - line_start + 1;
+                        }
+                        span.line_stop = line_no;
+                    }
+                    Token::AddressLit(address) => {
+                        if !validate_address(address) {
+                            return Err(TokenError::invalid_address_lit(address, &span));
+                        }
+                    }
+                    _ => (),
                 }
                 tokens.push(SpannedToken { token, span });
                 index += input.len() - output.len();
@@ -209,4 +226,33 @@ mod tests {
  "#
         );
     }
+
+    #[test]
+    fn test_spans() {
+        let raw = r#"
+            test
+            // test
+            test
+            /* test */
+            test
+            /* test
+            test */
+            test
+            "#;
+        let tokens = tokenize("test_path", raw).unwrap();
+        let mut line_indicies = vec![0];
+        for (i, c) in raw.chars().enumerate() {
+            if c == '\n' {
+                line_indicies.push(i + 1);
+            }
+        }
+        for token in tokens.iter() {
+            let token_raw = token.token.to_string();
+            let start = line_indicies.get(token.span.line_start - 1).unwrap();
+            let stop = line_indicies.get(token.span.line_stop - 1).unwrap();
+            let original = &raw[*start + token.span.col_start - 1..*stop + token.span.col_stop - 1];
+            assert_eq!(original, &token_raw);
+        }
+        println!("{}", serde_json::to_string_pretty(&tokens).unwrap());
+    }
 }

parser/src/tokenizer/token.rs

@@ -14,9 +14,10 @@
 // You should have received a copy of the GNU General Public License
 // along with the Leo library. If not, see <https://www.gnu.org/licenses/>.
 
+use serde::{Deserialize, Serialize};
 use std::fmt;
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum FormattedStringPart {
     Const(String),
     Container,
@@ -31,7 +32,7 @@ impl fmt::Display for FormattedStringPart {
     }
 }
 
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub enum Token {
     FormattedString(Vec<FormattedStringPart>),
     AddressLit(String),
@@ -186,8 +187,8 @@ impl fmt::Display for Token {
             AddressLit(s) => write!(f, "{}", s),
             Ident(s) => write!(f, "{}", s),
             Int(s) => write!(f, "{}", s),
-            CommentLine(s) => writeln!(f, "//{}", s),
-            CommentBlock(s) => write!(f, "/*{}*/ ", s),
+            CommentLine(s) => writeln!(f, "{}", s),
+            CommentBlock(s) => write!(f, "{}", s),
             Not => write!(f, "!"),
             NotEq => write!(f, "!="),
             And => write!(f, "&&"),