Merge pull request #4266 from charliermarsh/charlie/comments

Implement Tok::Comment

parser/python.lalrpop

@@ -1414,5 +1414,6 @@ extern {
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,
+        "#" => lexer::Tok::Comment,
     }
 }

parser/src/lexer.rs

@@ -399,13 +399,16 @@ where
     }
 
     /// Skip everything until end of line
-    fn lex_comment(&mut self) {
+    fn lex_comment(&mut self) -> LexResult {
+        let start_pos = self.get_pos();
         self.next_char();
         loop {
             match self.chr0 {
-                Some('\n') => return,
+                Some('\n') | None => {
+                    let end_pos = self.get_pos();
+                    return Ok((start_pos, Tok::Comment, end_pos));
+                }
                 Some(_) => {}
-                None => return,
             }
             self.next_char();
         }
@@ -690,7 +693,8 @@ where
                     tabs += 1;
                 }
                 Some('#') => {
-                    self.lex_comment();
+                    let comment = self.lex_comment()?;
+                    self.emit(comment);
                     spaces = 0;
                     tabs = 0;
                 }
@@ -832,7 +836,8 @@ where
                 self.emit(number);
             }
             '#' => {
-                self.lex_comment();
+                let comment = self.lex_comment()?;
+                self.emit(comment);
             }
             '"' | '\'' => {
                 let string = self.lex_string(false, false, false, false)?;
@@ -1350,7 +1355,7 @@ mod tests {
             fn $name() {
                 let source = format!(r"99232  # {}", $eol);
                 let tokens = lex_source(&source);
-                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
+                assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]);
             }
             )*
         }
@@ -1374,6 +1379,7 @@ mod tests {
                     tokens,
                     vec![
                         Tok::Int { value: BigInt::from(123) },
+                        Tok::Comment,
                         Tok::Newline,
                         Tok::Int { value: BigInt::from(456) },
                         Tok::Newline,

parser/src/parser.rs

@@ -5,9 +5,10 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
-use crate::lexer::LexResult;
+use crate::lexer::{LexResult, Tok};
 pub use crate::mode::Mode;
 use crate::{ast, error::ParseError, lexer, python};
+use itertools::Itertools;
 use std::iter;
 
 /*
@@ -74,7 +75,9 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
 pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
     let lxr = lexer::make_tokenizer(source);
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
-    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+    let tokenizer = iter::once(Ok(marker_token))
+        .chain(lxr)
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
 
     python::TopParser::new()
         .parse(tokenizer)
@@ -88,7 +91,9 @@ pub fn parse_tokens(
     source_path: &str,
 ) -> Result<ast::Mod, ParseError> {
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
-    let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
+    let tokenizer = iter::once(Ok(marker_token))
+        .chain(lxr)
+        .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
 
     python::TopParser::new()
         .parse(tokenizer)

parser/src/token.rs

@@ -25,6 +25,7 @@ pub enum Tok {
     Rsqb,
     Colon,
     Comma,
+    Comment,
     Semi,
     Plus,
     Minus,
@@ -155,6 +156,7 @@ impl fmt::Display for Tok {
             Rsqb => f.write_str("']'"),
             Colon => f.write_str("':'"),
             Comma => f.write_str("','"),
+            Comment => f.write_str("#"),
             Semi => f.write_str("';'"),
             Plus => f.write_str("'+'"),
             Minus => f.write_str("'-'"),