From d930e972128643d585b6a2c9622b3bf14dbde2ec Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Tue, 25 Jun 2024 13:40:07 +0530 Subject: [PATCH] Do not include newline for unterminated string range (#12017) ## Summary This PR updates the unterminated string error range to not include the final newline character. This is a follow-up to #12016 and required for #12019 This is not done for when the unterminated string goes till the end of file (not a newline character). The unterminated f-string range is correct. ### Why is this required for #12019 ? Because otherwise the token ranges will overlap. For example: ```py f"{" f"{foo!r" ``` Here, the re-lexing logic recovers from an unterminated f-string and thus emitting a `Newline` token for the one at the end of the first line. But, currently the `Unknown` and the `Newline` token would overlap because the `Unknown` token (unterminated string literal) range would include the newline character. ## Test Plan Update and validate the snapshot. --- crates/ruff_python_parser/src/lexer.rs | 4 +- ...id_syntax@f_string_unclosed_lbrace.py.snap | 78 +++++++++---------- ...y_concatenated_unterminated_string.py.snap | 10 +-- ...ated_unterminated_string_multiline.py.snap | 14 ++-- ...x@re_lexing__fstring_format_spec_1.py.snap | 12 ++- 5 files changed, 53 insertions(+), 65 deletions(-) diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 744e5212fb..cc04e79264 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -973,10 +973,10 @@ impl<'src> Lexer<'src> { } match ch { - Some('\r' | '\n') => { + Some(newline @ ('\r' | '\n')) => { return self.push_error(LexicalError::new( LexicalErrorType::UnclosedStringError, - self.token_range(), + self.token_range().sub_end(newline.text_len()), )); } Some(ch) if ch == quote => { diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap index 3f1856b37f..bdd816e505 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap @@ -11,15 +11,15 @@ Module( body: [ Expr( StmtExpr { - range: 0..5, + range: 0..4, value: FString( ExprFString { - range: 0..5, + range: 0..4, value: FStringValue { inner: Single( FString( FString { - range: 0..5, + range: 0..4, elements: [ Expression( FStringExpressionElement { @@ -52,19 +52,19 @@ Module( ), Expr( StmtExpr { - range: 5..15, + range: 5..14, value: FString( ExprFString { - range: 5..15, + range: 5..14, value: FStringValue { inner: Single( FString( FString { - range: 5..15, + range: 5..14, elements: [ Expression( FStringExpressionElement { - range: 7..15, + range: 7..14, expression: Name( ExprName { range: 8..11, @@ -93,15 +93,15 @@ Module( ), Expr( StmtExpr { - range: 15..24, + range: 15..23, value: FString( ExprFString { - range: 15..24, + range: 15..23, value: FStringValue { inner: Single( FString( FString { - range: 15..24, + range: 15..23, elements: [ Expression( FStringExpressionElement { @@ -148,7 +148,7 @@ Module( [ FString( FString { - range: 24..29, + range: 24..28, elements: [ Expression( FStringExpressionElement { @@ -213,19 +213,9 @@ Module( ``` ## Errors - | -1 | f"{" - | ____^ -2 | | f"{foo!r" - | |_^ Syntax Error: missing closing quote in string literal -3 | f"{foo=" -4 | f"{" - | - - | 1 | f"{" - | Syntax Error: f-string: unterminated string + | ^ Syntax Error: missing closing quote in string literal 2 | f"{foo!r" 3 | f"{foo=" | @@ -240,13 +230,19 @@ Module( | -1 | f"{" -2 | f"{foo!r" - | ________^ -3 | | f"{foo=" - | |_^ Syntax Error: missing closing quote in string literal -4 | f"{" -5 | f"""{""" +1 | f"{" + | Syntax Error: f-string: unterminated string +2 | f"{foo!r" +3 | f"{foo=" + | + + + | +1 | f"{" +2 | f"{foo!r" + | ^^ Syntax Error: missing closing quote in string literal +3 | f"{foo=" +4 | f"{" | @@ -288,13 +284,12 @@ Module( | -1 | f"{" -2 | f"{foo!r" -3 | f"{foo=" - | ________^ -4 | | f"{" - | |_^ Syntax Error: missing closing quote in string literal -5 | f"""{""" +1 | f"{" +2 | f"{foo!r" +3 | f"{foo=" + | ^ Syntax Error: missing closing quote in string literal +4 | f"{" +5 | f"""{""" | @@ -319,12 +314,11 @@ Module( | -2 | f"{foo!r" -3 | f"{foo=" -4 | f"{" - | ____^ -5 | | f"""{""" - | |_^ Syntax Error: missing closing quote in string literal +2 | f"{foo!r" +3 | f"{foo=" +4 | f"{" + | ^ Syntax Error: missing closing quote in string literal +5 | f"""{""" | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string.py.snap index ac5a0a2bb8..67ef835321 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string.py.snap @@ -153,12 +153,10 @@ Module( ## Errors | -1 | 'hello' 'world - | _________^ -2 | | 1 + 1 - | |_^ Syntax Error: missing closing quote in string literal -3 | 'hello' f'world {x} -4 | 2 + 2 +1 | 'hello' 'world + | ^^^^^^ Syntax Error: missing closing quote in string literal +2 | 1 + 1 +3 | 'hello' f'world {x} | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap index c29558f594..2091165382 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@implicitly_concatenated_unterminated_string_multiline.py.snap @@ -216,14 +216,12 @@ Module( | - 6 | ( - 7 | 'first' - 8 | 'second - | _____^ - 9 | | f'third' - | |_^ Syntax Error: missing closing quote in string literal -10 | ) -11 | 2 + 2 + 6 | ( + 7 | 'first' + 8 | 'second + | ^^^^^^^ Syntax Error: missing closing quote in string literal + 9 | f'third' +10 | ) | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__fstring_format_spec_1.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__fstring_format_spec_1.py.snap index 7fe73227d2..7251180b0c 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__fstring_format_spec_1.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__fstring_format_spec_1.py.snap @@ -324,13 +324,11 @@ Module( | -5 | f'middle {'string':\ -6 | 'format spec'} - | _____________________^ -7 | | - | |_^ Syntax Error: missing closing quote in string literal -8 | f'middle {'string':\\ -9 | 'format spec'} +5 | f'middle {'string':\ +6 | 'format spec'} + | ^^ Syntax Error: missing closing quote in string literal +7 | +8 | f'middle {'string':\\ |