From bb2abd1cbc237d5aa6d18e83af7b7ebf016ad7be Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 12 Jan 2023 22:34:04 +0200 Subject: [PATCH] Fixup parse_tokens after "Add NonLogicalNewline token" I only updated `parse()` to ignore `NonLogicalNewline`, didn't notice it's also needed in `parse_tokens()`. --- parser/src/parser.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/parser/src/parser.rs b/parser/src/parser.rs index 4d9b52b..fb18155 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -93,14 +93,7 @@ pub fn parse_located( location: Location, ) -> Result { let lxr = lexer::make_tokenizer_located(source, location); - let marker_token = (Default::default(), mode.to_marker(), Default::default()); - let tokenizer = iter::once(Ok(marker_token)) - .chain(lxr) - .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); - - python::TopParser::new() - .parse(tokenizer) - .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path)) + parse_tokens(lxr, mode, source_path) } // Parse a given token iterator. @@ -112,7 +105,7 @@ pub fn parse_tokens( let marker_token = (Default::default(), mode.to_marker(), Default::default()); let tokenizer = iter::once(Ok(marker_token)) .chain(lxr) - .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment(_))); + .filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline)); python::TopParser::new() .parse(tokenizer)