mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-31 23:57:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Include comment text in token

Browse source
This commit is contained in:
harupy 2023-01-06 23:29:20 +09:00
parent 2dfd053bed
commit a4a5366504
4 changed files with 11 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

11
parser/src/lexer.rs
View file

@ -444,16 +444,17 @@ where
/// Skip everything until end of line
fn lex_comment(&mut self) -> LexResult {
let start_pos = self.get_pos();
self.next_char();
let mut value = String::new();
value.push(self.next_char().unwrap());
loop {
match self.window[0] {
Some('\n') | None => {
let end_pos = self.get_pos();
return Ok((start_pos, Tok::Comment, end_pos));
return Ok((start_pos, Tok::Comment(value), end_pos));
}
Some(_) => {}
}
self.next_char();
value.push(self.next_char().unwrap());
}
}
@ -1266,7 +1267,7 @@ mod tests {
fn $name() {
let source = format!(r"99232 # {}", $eol);
let tokens = lex_source(&source);
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment, Tok::Newline]);
assert_eq!(tokens, vec![Tok::Int { value: BigInt::from(99232) }, Tok::Comment(format!("# {}", $eol)), Tok::Newline]);
}
)*
}
@ -1290,7 +1291,7 @@ mod tests {
tokens,
vec![
Tok::Int { value: BigInt::from(123) },
Tok::Comment,
Tok::Comment("# Foo".to_string()),
Tok::Newline,
Tok::Int { value: BigInt::from(456) },
Tok::Newline,