Use cursor offset for lexer checkpoint (#11734)

## Summary

This PR updates the lexer checkpoint to store the cursor offset instead
of cloning the cursor itself. This reduces the size of `LexerCheckpoint`
from 136 to 112 bytes and also removes the need for lifetime.

## Test Plan

`cargo insta test`
This commit is contained in:
Dhruv Manilawala 2024-06-04 14:13:57 +05:30 committed by GitHub
parent 6ffb96171a
commit 3b19df04d7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 18 additions and 14 deletions

View file

@ -1338,13 +1338,13 @@ impl<'src> Lexer<'src> {
} }
/// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`]. /// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`].
pub(crate) fn checkpoint(&self) -> LexerCheckpoint<'src> { pub(crate) fn checkpoint(&self) -> LexerCheckpoint {
LexerCheckpoint { LexerCheckpoint {
value: self.current_value.clone(), value: self.current_value.clone(),
current_kind: self.current_kind, current_kind: self.current_kind,
current_range: self.current_range, current_range: self.current_range,
current_flags: self.current_flags, current_flags: self.current_flags,
cursor: self.cursor.clone(), cursor_offset: self.offset(),
state: self.state, state: self.state,
nesting: self.nesting, nesting: self.nesting,
indentations_checkpoint: self.indentations.checkpoint(), indentations_checkpoint: self.indentations.checkpoint(),
@ -1355,13 +1355,13 @@ impl<'src> Lexer<'src> {
} }
/// Restore the lexer to the given checkpoint. /// Restore the lexer to the given checkpoint.
pub(crate) fn rewind(&mut self, checkpoint: LexerCheckpoint<'src>) { pub(crate) fn rewind(&mut self, checkpoint: LexerCheckpoint) {
let LexerCheckpoint { let LexerCheckpoint {
value, value,
current_kind, current_kind,
current_range, current_range,
current_flags, current_flags,
cursor, cursor_offset,
state, state,
nesting, nesting,
indentations_checkpoint, indentations_checkpoint,
@ -1370,6 +1370,10 @@ impl<'src> Lexer<'src> {
errors_position, errors_position,
} = checkpoint; } = checkpoint;
let mut cursor = Cursor::new(self.source);
// We preserve the previous char using this method.
cursor.skip_bytes(cursor_offset.to_usize());
self.current_value = value; self.current_value = value;
self.current_kind = current_kind; self.current_kind = current_kind;
self.current_range = current_range; self.current_range = current_range;
@ -1700,12 +1704,12 @@ pub(crate) enum TokenValue {
}, },
} }
pub(crate) struct LexerCheckpoint<'src> { pub(crate) struct LexerCheckpoint {
value: TokenValue, value: TokenValue,
current_kind: TokenKind, current_kind: TokenKind,
current_range: TextRange, current_range: TextRange,
current_flags: TokenFlags, current_flags: TokenFlags,
cursor: Cursor<'src>, cursor_offset: TextSize,
state: State, state: State,
nesting: u32, nesting: u32,
indentations_checkpoint: IndentationsCheckpoint, indentations_checkpoint: IndentationsCheckpoint,

View file

@ -609,7 +609,7 @@ impl<'src> Parser<'src> {
} }
/// Creates a checkpoint to which the parser can later return to using [`Self::rewind`]. /// Creates a checkpoint to which the parser can later return to using [`Self::rewind`].
fn checkpoint(&self) -> ParserCheckpoint<'src> { fn checkpoint(&self) -> ParserCheckpoint {
ParserCheckpoint { ParserCheckpoint {
tokens: self.tokens.checkpoint(), tokens: self.tokens.checkpoint(),
errors_position: self.errors.len(), errors_position: self.errors.len(),
@ -620,7 +620,7 @@ impl<'src> Parser<'src> {
} }
/// Restore the parser to the given checkpoint. /// Restore the parser to the given checkpoint.
fn rewind(&mut self, checkpoint: ParserCheckpoint<'src>) { fn rewind(&mut self, checkpoint: ParserCheckpoint) {
let ParserCheckpoint { let ParserCheckpoint {
tokens, tokens,
errors_position, errors_position,
@ -637,8 +637,8 @@ impl<'src> Parser<'src> {
} }
} }
struct ParserCheckpoint<'src> { struct ParserCheckpoint {
tokens: TokenSourceCheckpoint<'src>, tokens: TokenSourceCheckpoint,
errors_position: usize, errors_position: usize,
current_token_id: TokenId, current_token_id: TokenId,
prev_token_end: TextSize, prev_token_end: TextSize,

View file

@ -126,7 +126,7 @@ impl<'src> TokenSource<'src> {
} }
/// Creates a checkpoint to which the token source can later return to using [`Self::rewind`]. /// Creates a checkpoint to which the token source can later return to using [`Self::rewind`].
pub(crate) fn checkpoint(&self) -> TokenSourceCheckpoint<'src> { pub(crate) fn checkpoint(&self) -> TokenSourceCheckpoint {
TokenSourceCheckpoint { TokenSourceCheckpoint {
lexer_checkpoint: self.lexer.checkpoint(), lexer_checkpoint: self.lexer.checkpoint(),
tokens_position: self.tokens.len(), tokens_position: self.tokens.len(),
@ -135,7 +135,7 @@ impl<'src> TokenSource<'src> {
} }
/// Restore the token source to the given checkpoint. /// Restore the token source to the given checkpoint.
pub(crate) fn rewind(&mut self, checkpoint: TokenSourceCheckpoint<'src>) { pub(crate) fn rewind(&mut self, checkpoint: TokenSourceCheckpoint) {
let TokenSourceCheckpoint { let TokenSourceCheckpoint {
lexer_checkpoint, lexer_checkpoint,
tokens_position, tokens_position,
@ -168,8 +168,8 @@ impl<'src> TokenSource<'src> {
} }
} }
pub(crate) struct TokenSourceCheckpoint<'src> { pub(crate) struct TokenSourceCheckpoint {
lexer_checkpoint: LexerCheckpoint<'src>, lexer_checkpoint: LexerCheckpoint,
tokens_position: usize, tokens_position: usize,
comments_position: usize, comments_position: usize,
} }