mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-26 06:24:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add NonLogicalNewline token

Browse source 
This token is completely ignored by the parser, but it's useful for
other users of the lexer, such as the Ruff linter. For example, the
token is helpful for a "trailing comma" lint.

The same idea exists in Python's `tokenize` module - there is a NEWLINE
token (logical newline), and a NL token (non-logical newline).

Fixes #4385.
This commit is contained in:
Ran Benita 2023-01-12 14:53:55 +02:00
parent 4f1e7c6291
commit 674eeec29c
3 changed files with 84 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
parser/src/parser.rs
View file

@ -96,7 +96,7 @@ pub fn parse_located(
let marker_token = (Default::default(), mode.to_marker(), Default::default());
let tokenizer = iter::once(Ok(marker_token))
.chain(lxr)
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. }));
.filter_ok(|(_, tok, _)| !matches!(tok, Tok::Comment { .. } | Tok::NonLogicalNewline));
python::TopParser::new()
.parse(tokenizer)