mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-22 04:25:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Always emit non-logical newlines for 'empty' lines (#27)

Browse source
This commit is contained in:
Charlie Marsh 2023-05-15 14:13:05 -04:00 • committed by GitHub
parent 27e3873dc2
commit 10dda125ff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 12 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

16
parser/src/lexer.rs
View file

@ -641,7 +641,10 @@ where
} }
Some('\n' | '\r') => { Some('\n' | '\r') => {
// Empty line! // Empty line!
let tok_start = self.get_pos();
self.next_char(); self.next_char();
let tok_end = self.get_pos();
self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end)));
spaces = 0; spaces = 0;
tabs = 0; tabs = 0;
} }
@ -1139,7 +1142,7 @@ where
return Err(LexicalError { return Err(LexicalError {
error: LexicalErrorType::LineContinuationError, error: LexicalErrorType::LineContinuationError,
location: self.get_pos(), location: self.get_pos(),
}) });
} }
} }
@ -1521,6 +1524,7 @@ mod tests {
Tok::Return, Tok::Return,
Tok::Int { value: BigInt::from(99) }, Tok::Int { value: BigInt::from(99) },
Tok::Newline, Tok::Newline,
Tok::NonLogicalNewline,
Tok::Dedent, Tok::Dedent,
] ]
); );
@ -1560,10 +1564,12 @@ mod tests {
}, },
Tok::Colon, Tok::Colon,
Tok::Newline, Tok::Newline,
Tok::NonLogicalNewline,
Tok::Indent, Tok::Indent,
Tok::Return, Tok::Return,
Tok::Int { value: BigInt::from(99) }, Tok::Int { value: BigInt::from(99) },
Tok::Newline, Tok::Newline,
Tok::NonLogicalNewline,
Tok::Dedent, Tok::Dedent,
Tok::Dedent, Tok::Dedent,
] ]
@ -1598,10 +1604,12 @@ mod tests {
}, },
Tok::Colon, Tok::Colon,
Tok::Newline, Tok::Newline,
Tok::NonLogicalNewline,
Tok::Indent, Tok::Indent,
Tok::Return, Tok::Return,
Tok::Int { value: BigInt::from(99) }, Tok::Int { value: BigInt::from(99) },
Tok::Newline, Tok::Newline,
Tok::NonLogicalNewline,
Tok::Dedent, Tok::Dedent,
Tok::Dedent, Tok::Dedent,
] ]
@ -1722,15 +1730,15 @@ mod tests {
#[test] #[test]
#[cfg(feature = "full-lexer")] #[cfg(feature = "full-lexer")]
fn test_logical_newline_line_comment() { fn test_logical_newline_line_comment() {
let source = "#Hello\n#World"; let source = "#Hello\n#World\n";
let tokens = lex_source(source); let tokens = lex_source(source);
assert_eq!( assert_eq!(
tokens, tokens,
vec![ vec![
Tok::Comment("#Hello".to_owned()), Tok::Comment("#Hello".to_owned()),
// tokenize.py does put an NL here... Tok::NonLogicalNewline,
Tok::Comment("#World".to_owned()), Tok::Comment("#World".to_owned()),
// ... and here, but doesn't seem very useful. Tok::NonLogicalNewline,
] ]
); );
} }