mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-19 02:55:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Include comment text in token

Browse source
This commit is contained in:
harupy 2023-01-06 23:29:20 +09:00
parent 60cbfd3a94
commit 1a28e49adc
4 changed files with 11 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

4
parser/src/parser.rs
View file

@ -96,7 +96,7 @@ pub fn parse_located(
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
python::TopParser::new()
.parse(tokenizer)
@ -112,7 +112,7 @@ pub fn parse_tokens(
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment));
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
python::TopParser::new()
.parse(tokenizer)