mirror of
https://github.com/astral-sh/ruff.git
synced 2025-09-27 04:19:43 +00:00
Use cursor offset for lexer checkpoint (#11734)
## Summary This PR updates the lexer checkpoint to store the cursor offset instead of cloning the cursor itself. This reduces the size of `LexerCheckpoint` from 136 to 112 bytes and also removes the need for lifetime. ## Test Plan `cargo insta test`
This commit is contained in:
parent
6ffb96171a
commit
3b19df04d7
3 changed files with 18 additions and 14 deletions
|
@ -1338,13 +1338,13 @@ impl<'src> Lexer<'src> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`].
|
/// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`].
|
||||||
pub(crate) fn checkpoint(&self) -> LexerCheckpoint<'src> {
|
pub(crate) fn checkpoint(&self) -> LexerCheckpoint {
|
||||||
LexerCheckpoint {
|
LexerCheckpoint {
|
||||||
value: self.current_value.clone(),
|
value: self.current_value.clone(),
|
||||||
current_kind: self.current_kind,
|
current_kind: self.current_kind,
|
||||||
current_range: self.current_range,
|
current_range: self.current_range,
|
||||||
current_flags: self.current_flags,
|
current_flags: self.current_flags,
|
||||||
cursor: self.cursor.clone(),
|
cursor_offset: self.offset(),
|
||||||
state: self.state,
|
state: self.state,
|
||||||
nesting: self.nesting,
|
nesting: self.nesting,
|
||||||
indentations_checkpoint: self.indentations.checkpoint(),
|
indentations_checkpoint: self.indentations.checkpoint(),
|
||||||
|
@ -1355,13 +1355,13 @@ impl<'src> Lexer<'src> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restore the lexer to the given checkpoint.
|
/// Restore the lexer to the given checkpoint.
|
||||||
pub(crate) fn rewind(&mut self, checkpoint: LexerCheckpoint<'src>) {
|
pub(crate) fn rewind(&mut self, checkpoint: LexerCheckpoint) {
|
||||||
let LexerCheckpoint {
|
let LexerCheckpoint {
|
||||||
value,
|
value,
|
||||||
current_kind,
|
current_kind,
|
||||||
current_range,
|
current_range,
|
||||||
current_flags,
|
current_flags,
|
||||||
cursor,
|
cursor_offset,
|
||||||
state,
|
state,
|
||||||
nesting,
|
nesting,
|
||||||
indentations_checkpoint,
|
indentations_checkpoint,
|
||||||
|
@ -1370,6 +1370,10 @@ impl<'src> Lexer<'src> {
|
||||||
errors_position,
|
errors_position,
|
||||||
} = checkpoint;
|
} = checkpoint;
|
||||||
|
|
||||||
|
let mut cursor = Cursor::new(self.source);
|
||||||
|
// We preserve the previous char using this method.
|
||||||
|
cursor.skip_bytes(cursor_offset.to_usize());
|
||||||
|
|
||||||
self.current_value = value;
|
self.current_value = value;
|
||||||
self.current_kind = current_kind;
|
self.current_kind = current_kind;
|
||||||
self.current_range = current_range;
|
self.current_range = current_range;
|
||||||
|
@ -1700,12 +1704,12 @@ pub(crate) enum TokenValue {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct LexerCheckpoint<'src> {
|
pub(crate) struct LexerCheckpoint {
|
||||||
value: TokenValue,
|
value: TokenValue,
|
||||||
current_kind: TokenKind,
|
current_kind: TokenKind,
|
||||||
current_range: TextRange,
|
current_range: TextRange,
|
||||||
current_flags: TokenFlags,
|
current_flags: TokenFlags,
|
||||||
cursor: Cursor<'src>,
|
cursor_offset: TextSize,
|
||||||
state: State,
|
state: State,
|
||||||
nesting: u32,
|
nesting: u32,
|
||||||
indentations_checkpoint: IndentationsCheckpoint,
|
indentations_checkpoint: IndentationsCheckpoint,
|
||||||
|
|
|
@ -609,7 +609,7 @@ impl<'src> Parser<'src> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a checkpoint to which the parser can later return to using [`Self::rewind`].
|
/// Creates a checkpoint to which the parser can later return to using [`Self::rewind`].
|
||||||
fn checkpoint(&self) -> ParserCheckpoint<'src> {
|
fn checkpoint(&self) -> ParserCheckpoint {
|
||||||
ParserCheckpoint {
|
ParserCheckpoint {
|
||||||
tokens: self.tokens.checkpoint(),
|
tokens: self.tokens.checkpoint(),
|
||||||
errors_position: self.errors.len(),
|
errors_position: self.errors.len(),
|
||||||
|
@ -620,7 +620,7 @@ impl<'src> Parser<'src> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restore the parser to the given checkpoint.
|
/// Restore the parser to the given checkpoint.
|
||||||
fn rewind(&mut self, checkpoint: ParserCheckpoint<'src>) {
|
fn rewind(&mut self, checkpoint: ParserCheckpoint) {
|
||||||
let ParserCheckpoint {
|
let ParserCheckpoint {
|
||||||
tokens,
|
tokens,
|
||||||
errors_position,
|
errors_position,
|
||||||
|
@ -637,8 +637,8 @@ impl<'src> Parser<'src> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct ParserCheckpoint<'src> {
|
struct ParserCheckpoint {
|
||||||
tokens: TokenSourceCheckpoint<'src>,
|
tokens: TokenSourceCheckpoint,
|
||||||
errors_position: usize,
|
errors_position: usize,
|
||||||
current_token_id: TokenId,
|
current_token_id: TokenId,
|
||||||
prev_token_end: TextSize,
|
prev_token_end: TextSize,
|
||||||
|
|
|
@ -126,7 +126,7 @@ impl<'src> TokenSource<'src> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a checkpoint to which the token source can later return to using [`Self::rewind`].
|
/// Creates a checkpoint to which the token source can later return to using [`Self::rewind`].
|
||||||
pub(crate) fn checkpoint(&self) -> TokenSourceCheckpoint<'src> {
|
pub(crate) fn checkpoint(&self) -> TokenSourceCheckpoint {
|
||||||
TokenSourceCheckpoint {
|
TokenSourceCheckpoint {
|
||||||
lexer_checkpoint: self.lexer.checkpoint(),
|
lexer_checkpoint: self.lexer.checkpoint(),
|
||||||
tokens_position: self.tokens.len(),
|
tokens_position: self.tokens.len(),
|
||||||
|
@ -135,7 +135,7 @@ impl<'src> TokenSource<'src> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restore the token source to the given checkpoint.
|
/// Restore the token source to the given checkpoint.
|
||||||
pub(crate) fn rewind(&mut self, checkpoint: TokenSourceCheckpoint<'src>) {
|
pub(crate) fn rewind(&mut self, checkpoint: TokenSourceCheckpoint) {
|
||||||
let TokenSourceCheckpoint {
|
let TokenSourceCheckpoint {
|
||||||
lexer_checkpoint,
|
lexer_checkpoint,
|
||||||
tokens_position,
|
tokens_position,
|
||||||
|
@ -168,8 +168,8 @@ impl<'src> TokenSource<'src> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) struct TokenSourceCheckpoint<'src> {
|
pub(crate) struct TokenSourceCheckpoint {
|
||||||
lexer_checkpoint: LexerCheckpoint<'src>,
|
lexer_checkpoint: LexerCheckpoint,
|
||||||
tokens_position: usize,
|
tokens_position: usize,
|
||||||
comments_position: usize,
|
comments_position: usize,
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue