mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-12 23:55:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Include comment text in token

Browse source
This commit is contained in:
harupy 2023-01-06 23:29:20 +09:00
parent 60cbfd3a94
commit 1a28e49adc
4 changed files with 11 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
parser/python.lalrpop
View file

@ -1463,6 +1463,6 @@ extern {
name => lexer::Tok::Name { name: <String> }, name => lexer::Tok::Name { name: <String> },
"\n" => lexer::Tok::Newline, "\n" => lexer::Tok::Newline,
";" => lexer::Tok::Semi, ";" => lexer::Tok::Semi,
"#" => lexer::Tok::Comment, "#" => lexer::Tok::Comment(_),
} }
} }

11
parser/src/lexer.rs
View file

@ -444,16 +444,17 @@ where
/// Skip everything until end of line /// Skip everything until end of line
fn lex_comment(&mut self) -> LexResult { fn lex_comment(&mut self) -> LexResult {
let start_pos = self.get_pos(); let start_pos = self.get_pos();
self.next_char(); let mut value = String::new();
value.push(self.next_char().unwrap());
loop { loop {
match self.window[0] { match self.window[0] {
Some('\n') | None => { Some('\n') | None => {
let end_pos = self.get_pos(); let end_pos = self.get_pos();
return Ok((start_pos, Tok::Comment, end_pos)); return Ok((start_pos, Tok::Comment(value), end_pos));
} }
Some(_) => {} Some(_) => {}
} }
self.next_char(); value.push(self.next_char().unwrap());
} }
} }
@ -1266,7 +1267,7 @@ mod tests {
fn $name() { fn $name() {
let source = format!(r"99232 # {}", $eol); let source = format!(r"99232 # {}", $eol);
let tokens = lex_source(&source); let tokens = lex_source(&source);
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]); assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment(format!("# {}", $eol)), Tok::Newline]);
} }
)* )*
} }
@ -1290,7 +1291,7 @@ mod tests {
tokens, tokens,
vec![ vec![
Tok::Int { value: BigInt::from(123) }, Tok::Int { value: BigInt::from(123) },
Tok::Comment, Tok::Comment("# Foo".to_string()),
Tok::Newline, Tok::Newline,
Tok::Int { value: BigInt::from(456) }, Tok::Int { value: BigInt::from(456) },
Tok::Newline, Tok::Newline,

4
parser/src/parser.rs
View file

@ -96,7 +96,7 @@ pub fn parse_located(
let marker_token = (Default::default(), mode.to_marker(), Default::default()); let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token)) let tokenizer = iter::once(Ok(marker_token))
.chain(lxr) .chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment)); .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
python::TopParser::new() python::TopParser::new()
.parse(tokenizer) .parse(tokenizer)
@ -112,7 +112,7 @@ pub fn parse_tokens(
let marker_token = (Default::default(), mode.to_marker(), Default::default()); let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token)) let tokenizer = iter::once(Ok(marker_token))
.chain(lxr) .chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment)); .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
python::TopParser::new() python::TopParser::new()
.parse(tokenizer) .parse(tokenizer)

4
parser/src/token.rs
View file

@ -37,7 +37,7 @@ pub enum Tok {
Rsqb, Rsqb,
Colon, Colon,
Comma, Comma,
Comment, Comment(String),
Semi, Semi,
Plus, Plus,
Minus, Minus,
@ -148,7 +148,7 @@ impl fmt::Display for Tok {
Rsqb => f.write_str("']'"), Rsqb => f.write_str("']'"),
Colon => f.write_str("':'"), Colon => f.write_str("':'"),
Comma => f.write_str("','"), Comma => f.write_str("','"),
Comment => f.write_str("#"), Comment(value) => f.write_str(value),
Semi => f.write_str("';'"), Semi => f.write_str("';'"),
Plus => f.write_str("'+'"), Plus => f.write_str("'+'"),
Minus => f.write_str("'-'"), Minus => f.write_str("'-'"),