From 10dda125fff77c1b86e02f04aa680088e7ceade0 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 15 May 2023 14:13:05 -0400 Subject: [PATCH] Always emit non-logical newlines for 'empty' lines (#27) --- parser/src/lexer.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index ad18b61..b85ddd7 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -641,7 +641,10 @@ where } Some('\n' | '\r') => { // Empty line! + let tok_start = self.get_pos(); self.next_char(); + let tok_end = self.get_pos(); + self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end))); spaces = 0; tabs = 0; } @@ -1139,7 +1142,7 @@ where return Err(LexicalError { error: LexicalErrorType::LineContinuationError, location: self.get_pos(), - }) + }); } } @@ -1521,6 +1524,7 @@ mod tests { Tok::Return, Tok::Int { value: BigInt::from(99) }, Tok::Newline, + Tok::NonLogicalNewline, Tok::Dedent, ] ); @@ -1560,10 +1564,12 @@ mod tests { }, Tok::Colon, Tok::Newline, + Tok::NonLogicalNewline, Tok::Indent, Tok::Return, Tok::Int { value: BigInt::from(99) }, Tok::Newline, + Tok::NonLogicalNewline, Tok::Dedent, Tok::Dedent, ] @@ -1598,10 +1604,12 @@ mod tests { }, Tok::Colon, Tok::Newline, + Tok::NonLogicalNewline, Tok::Indent, Tok::Return, Tok::Int { value: BigInt::from(99) }, Tok::Newline, + Tok::NonLogicalNewline, Tok::Dedent, Tok::Dedent, ] @@ -1722,15 +1730,15 @@ mod tests { #[test] #[cfg(feature = "full-lexer")] fn test_logical_newline_line_comment() { - let source = "#Hello\n#World"; + let source = "#Hello\n#World\n"; let tokens = lex_source(source); assert_eq!( tokens, vec![ Tok::Comment("#Hello".to_owned()), - // tokenize.py does put an NL here... + Tok::NonLogicalNewline, Tok::Comment("#World".to_owned()), - // ... and here, but doesn't seem very useful. + Tok::NonLogicalNewline, ] ); }