diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 41724a13dd..aa4eebdc68 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -1338,13 +1338,13 @@ impl<'src> Lexer<'src> { } /// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`]. - pub(crate) fn checkpoint(&self) -> LexerCheckpoint<'src> { + pub(crate) fn checkpoint(&self) -> LexerCheckpoint { LexerCheckpoint { value: self.current_value.clone(), current_kind: self.current_kind, current_range: self.current_range, current_flags: self.current_flags, - cursor: self.cursor.clone(), + cursor_offset: self.offset(), state: self.state, nesting: self.nesting, indentations_checkpoint: self.indentations.checkpoint(), @@ -1355,13 +1355,13 @@ impl<'src> Lexer<'src> { } /// Restore the lexer to the given checkpoint. - pub(crate) fn rewind(&mut self, checkpoint: LexerCheckpoint<'src>) { + pub(crate) fn rewind(&mut self, checkpoint: LexerCheckpoint) { let LexerCheckpoint { value, current_kind, current_range, current_flags, - cursor, + cursor_offset, state, nesting, indentations_checkpoint, @@ -1370,6 +1370,10 @@ impl<'src> Lexer<'src> { errors_position, } = checkpoint; + let mut cursor = Cursor::new(self.source); + // We preserve the previous char using this method. + cursor.skip_bytes(cursor_offset.to_usize()); + self.current_value = value; self.current_kind = current_kind; self.current_range = current_range; @@ -1700,12 +1704,12 @@ pub(crate) enum TokenValue { }, } -pub(crate) struct LexerCheckpoint<'src> { +pub(crate) struct LexerCheckpoint { value: TokenValue, current_kind: TokenKind, current_range: TextRange, current_flags: TokenFlags, - cursor: Cursor<'src>, + cursor_offset: TextSize, state: State, nesting: u32, indentations_checkpoint: IndentationsCheckpoint, diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index f1b240cfd6..c04875c45b 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -609,7 +609,7 @@ impl<'src> Parser<'src> { } /// Creates a checkpoint to which the parser can later return to using [`Self::rewind`]. - fn checkpoint(&self) -> ParserCheckpoint<'src> { + fn checkpoint(&self) -> ParserCheckpoint { ParserCheckpoint { tokens: self.tokens.checkpoint(), errors_position: self.errors.len(), @@ -620,7 +620,7 @@ impl<'src> Parser<'src> { } /// Restore the parser to the given checkpoint. - fn rewind(&mut self, checkpoint: ParserCheckpoint<'src>) { + fn rewind(&mut self, checkpoint: ParserCheckpoint) { let ParserCheckpoint { tokens, errors_position, @@ -637,8 +637,8 @@ impl<'src> Parser<'src> { } } -struct ParserCheckpoint<'src> { - tokens: TokenSourceCheckpoint<'src>, +struct ParserCheckpoint { + tokens: TokenSourceCheckpoint, errors_position: usize, current_token_id: TokenId, prev_token_end: TextSize, diff --git a/crates/ruff_python_parser/src/token_source.rs b/crates/ruff_python_parser/src/token_source.rs index 005c5ff38d..79e8a2094f 100644 --- a/crates/ruff_python_parser/src/token_source.rs +++ b/crates/ruff_python_parser/src/token_source.rs @@ -126,7 +126,7 @@ impl<'src> TokenSource<'src> { } /// Creates a checkpoint to which the token source can later return to using [`Self::rewind`]. - pub(crate) fn checkpoint(&self) -> TokenSourceCheckpoint<'src> { + pub(crate) fn checkpoint(&self) -> TokenSourceCheckpoint { TokenSourceCheckpoint { lexer_checkpoint: self.lexer.checkpoint(), tokens_position: self.tokens.len(), @@ -135,7 +135,7 @@ impl<'src> TokenSource<'src> { } /// Restore the token source to the given checkpoint. - pub(crate) fn rewind(&mut self, checkpoint: TokenSourceCheckpoint<'src>) { + pub(crate) fn rewind(&mut self, checkpoint: TokenSourceCheckpoint) { let TokenSourceCheckpoint { lexer_checkpoint, tokens_position, @@ -168,8 +168,8 @@ impl<'src> TokenSource<'src> { } } -pub(crate) struct TokenSourceCheckpoint<'src> { - lexer_checkpoint: LexerCheckpoint<'src>, +pub(crate) struct TokenSourceCheckpoint { + lexer_checkpoint: LexerCheckpoint, tokens_position: usize, comments_position: usize, }