mirror of
https://github.com/RustPython/Parser.git
synced 2025-07-12 07:35:16 +00:00
Merge pull request #4266 from charliermarsh/charlie/comments
Implement Tok::Comment
This commit is contained in:
commit
cc084b4fec
4 changed files with 23 additions and 9 deletions
|
@ -1414,5 +1414,6 @@ extern {
|
|||
name => lexer::Tok::Name { name: <String> },
|
||||
"\n" => lexer::Tok::Newline,
|
||||
";" => lexer::Tok::Semi,
|
||||
"#" => lexer::Tok::Comment,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -399,13 +399,16 @@ where
|
|||
}
|
||||
|
||||
/// Skip everything until end of line
|
||||
fn lex_comment(&mut self) {
|
||||
fn lex_comment(&mut self) -> LexResult {
|
||||
let start_pos = self.get_pos();
|
||||
self.next_char();
|
||||
loop {
|
||||
match self.chr0 {
|
||||
Some('\n') => return,
|
||||
Some('\n') | None => {
|
||||
let end_pos = self.get_pos();
|
||||
return Ok((start_pos, Tok::Comment, end_pos));
|
||||
}
|
||||
Some(_) => {}
|
||||
None => return,
|
||||
}
|
||||
self.next_char();
|
||||
}
|
||||
|
@ -690,7 +693,8 @@ where
|
|||
tabs += 1;
|
||||
}
|
||||
Some('#') => {
|
||||
self.lex_comment();
|
||||
let comment = self.lex_comment()?;
|
||||
self.emit(comment);
|
||||
spaces = 0;
|
||||
tabs = 0;
|
||||
}
|
||||
|
@ -832,7 +836,8 @@ where
|
|||
self.emit(number);
|
||||
}
|
||||
'#' => {
|
||||
self.lex_comment();
|
||||
let comment = self.lex_comment()?;
|
||||
self.emit(comment);
|
||||
}
|
||||
'"' | '\'' => {
|
||||
let string = self.lex_string(false, false, false, false)?;
|
||||
|
@ -1350,7 +1355,7 @@ mod tests {
|
|||
fn $name() {
|
||||
let source = format!(r"99232 # {}", $eol);
|
||||
let tokens = lex_source(&source);
|
||||
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Newline]);
|
||||
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]);
|
||||
}
|
||||
)*
|
||||
}
|
||||
|
@ -1374,6 +1379,7 @@ mod tests {
|
|||
tokens,
|
||||
vec![
|
||||
Tok::Int { value: BigInt::from(123) },
|
||||
Tok::Comment,
|
||||
Tok::Newline,
|
||||
Tok::Int { value: BigInt::from(456) },
|
||||
Tok::Newline,
|
||||
|
|
|
@ -5,9 +5,10 @@
|
|||
//! parse a whole program, a single statement, or a single
|
||||
//! expression.
|
||||
|
||||
use crate::lexer::LexResult;
|
||||
use crate::lexer::{LexResult, Tok};
|
||||
pub use crate::mode::Mode;
|
||||
use crate::{ast, error::ParseError, lexer, python};
|
||||
use itertools::Itertools;
|
||||
use std::iter;
|
||||
|
||||
/*
|
||||
|
@ -74,7 +75,9 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
|
|||
pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
|
||||
let lxr = lexer::make_tokenizer(source);
|
||||
let marker_token = (Default::default(), mode.to_marker(), Default::default());
|
||||
let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
|
||||
let tokenizer = iter::once(Ok(marker_token))
|
||||
.chain(lxr)
|
||||
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
|
||||
|
||||
python::TopParser::new()
|
||||
.parse(tokenizer)
|
||||
|
@ -88,7 +91,9 @@ pub fn parse_tokens(
|
|||
source_path: &str,
|
||||
) -> Result<ast::Mod, ParseError> {
|
||||
let marker_token = (Default::default(), mode.to_marker(), Default::default());
|
||||
let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
|
||||
let tokenizer = iter::once(Ok(marker_token))
|
||||
.chain(lxr)
|
||||
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
|
||||
|
||||
python::TopParser::new()
|
||||
.parse(tokenizer)
|
||||
|
|
|
@ -25,6 +25,7 @@ pub enum Tok {
|
|||
Rsqb,
|
||||
Colon,
|
||||
Comma,
|
||||
Comment,
|
||||
Semi,
|
||||
Plus,
|
||||
Minus,
|
||||
|
@ -155,6 +156,7 @@ impl fmt::Display for Tok {
|
|||
Rsqb => f.write_str("']'"),
|
||||
Colon => f.write_str("':'"),
|
||||
Comma => f.write_str("','"),
|
||||
Comment => f.write_str("#"),
|
||||
Semi => f.write_str("';'"),
|
||||
Plus => f.write_str("'+'"),
|
||||
Minus => f.write_str("'-'"),
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue