mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-04 02:39:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #4443 from bluetech/non-logical-newline-token-fixup

Browse source 
Fixup parse_tokens after "Add NonLogicalNewline token"
This commit is contained in:
Jim Fasarakis-Hilliard 2023-01-14 12:49:04 +02:00 • committed by GitHub
commit ac4d3c076c
1 changed files with 2 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

11
parser/src/parser.rs
View file

@ -93,14 +93,7 @@ pub fn parse_located(
location: Location, location: Location,
) -> Result<ast::Mod, ParseError> { ) -> Result<ast::Mod, ParseError> {
let lxr = lexer::make_tokenizer_located(source, location); let lxr = lexer::make_tokenizer_located(source, location);
let marker_token = (Default::default(), mode.to_marker(), Default::default()); parse_tokens(lxr, mode, source_path)
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(tokenizer)
.map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
} }
// Parse a given token iterator. // Parse a given token iterator.
@ -112,7 +105,7 @@ pub fn parse_tokens(
let marker_token = (Default::default(), mode.to_marker(), Default::default()); let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token)) let tokenizer = iter::once(Ok(marker_token))
.chain(lxr) .chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_))); .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new() python::TopParser::new()
.parse(tokenizer) .parse(tokenizer)