Use correct range to highlight line continuation error (#12016)

## Summary

This PR fixes the range highlighted for the line continuation error.

Previously, it would highlight an incorrect range:
```
1 | call(a, b, \\\
  |           ^^ Syntax Error: unexpected character after line continuation character
2 | 
3 | def bar():
  |
```

And now:
```
  |
1 | call(a, b, \\\
  |             ^ Syntax Error: unexpected character after line continuation character
2 | 
3 | def bar():
  |
```

This is implemented by avoiding to update the token range for the
`Unknown` token which is emitted when there's a lexical error. Instead,
the `push_error` helper method will be responsible to update the range
to the error location.

This actually becomes a requirement which can be seen in follow-up PRs.

## Test Plan

Update and validate the snapshot.
This commit is contained in:
Dhruv Manilawala 2024-06-25 13:35:24 +05:30 committed by GitHub
parent 692309ebd7
commit 9c1b6ec411
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 19 additions and 14 deletions

View file

@ -133,13 +133,24 @@ impl<'src> Lexer<'src> {
std::mem::take(&mut self.current_value) std::mem::take(&mut self.current_value)
} }
/// Helper function to push the given error, updating the current range with the error location
/// and return the [`TokenKind::Unknown`] token.
fn push_error(&mut self, error: LexicalError) -> TokenKind {
self.current_range = error.location();
self.errors.push(error);
TokenKind::Unknown
}
/// Lex the next token. /// Lex the next token.
pub fn next_token(&mut self) -> TokenKind { pub fn next_token(&mut self) -> TokenKind {
self.cursor.start_token(); self.cursor.start_token();
self.current_value = TokenValue::None; self.current_value = TokenValue::None;
self.current_flags = TokenFlags::empty(); self.current_flags = TokenFlags::empty();
self.current_kind = self.lex_token(); self.current_kind = self.lex_token();
self.current_range = self.token_range(); // For `Unknown` token, the `push_error` method updates the current range.
if !matches!(self.current_kind, TokenKind::Unknown) {
self.current_range = self.token_range();
}
self.current_kind self.current_kind
} }
@ -236,7 +247,7 @@ impl<'src> Lexer<'src> {
} else if !self.cursor.eat_char('\n') { } else if !self.cursor.eat_char('\n') {
return Some(self.push_error(LexicalError::new( return Some(self.push_error(LexicalError::new(
LexicalErrorType::LineContinuationError, LexicalErrorType::LineContinuationError,
self.token_range(), TextRange::at(self.offset(), self.cursor.first().text_len()),
))); )));
} }
indentation = Indentation::root(); indentation = Indentation::root();
@ -328,7 +339,7 @@ impl<'src> Lexer<'src> {
} else if !self.cursor.eat_char('\n') { } else if !self.cursor.eat_char('\n') {
return Err(LexicalError::new( return Err(LexicalError::new(
LexicalErrorType::LineContinuationError, LexicalErrorType::LineContinuationError,
self.token_range(), TextRange::at(self.offset(), self.cursor.first().text_len()),
)); ));
} }
} }
@ -1464,12 +1475,6 @@ impl<'src> Lexer<'src> {
self.token_range().start() self.token_range().start()
} }
/// Helper function to push the given error and return the [`TokenKind::Unknown`] token.
fn push_error(&mut self, error: LexicalError) -> TokenKind {
self.errors.push(error);
TokenKind::Unknown
}
/// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`]. /// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`].
pub(crate) fn checkpoint(&self) -> LexerCheckpoint { pub(crate) fn checkpoint(&self) -> LexerCheckpoint {
LexerCheckpoint { LexerCheckpoint {

View file

@ -11,10 +11,10 @@ Module(
body: [ body: [
Expr( Expr(
StmtExpr { StmtExpr {
range: 0..13, range: 0..14,
value: Call( value: Call(
ExprCall { ExprCall {
range: 0..13, range: 0..14,
func: Name( func: Name(
ExprName { ExprName {
range: 0..4, range: 0..4,
@ -23,7 +23,7 @@ Module(
}, },
), ),
arguments: Arguments { arguments: Arguments {
range: 4..13, range: 4..14,
args: [ args: [
Name( Name(
ExprName { ExprName {
@ -82,7 +82,7 @@ Module(
| |
1 | call(a, b, \\\ 1 | call(a, b, \\\
| ^^ Syntax Error: unexpected character after line continuation character | ^ Syntax Error: unexpected character after line continuation character
2 | 2 |
3 | def bar(): 3 | def bar():
| |
@ -90,7 +90,7 @@ Module(
| |
1 | call(a, b, \\\ 1 | call(a, b, \\\
| ^ Syntax Error: unexpected character after line continuation character | ^ Syntax Error: unexpected character after line continuation character
2 | 2 |
3 | def bar(): 3 | def bar():
| |