mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-08 05:35:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4443 from bluetech/non-logical-newline-token-fixup

Browse source 
Fixup parse_tokens after "Add NonLogicalNewline token"
This commit is contained in:
Jim Fasarakis-Hilliard 2023-01-14 12:49:04 +02:00 • committed by GitHub
commit 163cb5cd67
1 changed files with 2 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
parser/src/parser.rs
View file

@ -93,14 +93,7 @@ pub fn parse_located(
location: Location,
) -> Result<ast::Mod, ParseError> {
let lxr = lexer::make_tokenizer_located(source, location);
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(tokenizer)
.map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
parse_tokens(lxr, mode, source_path)
}
// Parse a given token iterator.
@ -112,7 +105,7 @@ pub fn parse_tokens(
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_)));
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(tokenizer)