mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-08 05:35:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixup parse_tokens after "Add NonLogicalNewline token"

Browse source 
I only updated `parse()` to ignore `NonLogicalNewline`, didn't notice
it's also needed in `parse_tokens()`.
This commit is contained in:
Ran Benita 2023-01-12 22:34:04 +02:00
parent 674eeec29c
commit e5fe037e38
1 changed files with 2 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
parser/src/parser.rs
View file

@ -93,14 +93,7 @@ pub fn parse_located(
location: Location,
) -> Result<ast::Mod, ParseError> {
let lxr = lexer::make_tokenizer_located(source, location);
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(tokenizer)
.map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
parse_tokens(lxr, mode, source_path)
}
// Parse a given token iterator.
@ -112,7 +105,7 @@ pub fn parse_tokens(
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(tokenizer)