mirror of
https://github.com/astral-sh/ruff.git
synced 2025-09-28 04:45:01 +00:00
Add support for multi-character operator tokens to SimpleTokenizer
(#6563)
## Summary Allows for proper lexing of tokens like `->`. The main challenge is to ensure that our forward and backwards representations are the same for cases like `===`. Specifically, we want that to lex as `==` followed by `=` regardless of whether it's a forwards or backwards lex. To do so, we identify the range of the sequential characters (the full span of `===`), lex it forwards, then return the last token. ## Test Plan `cargo test`
This commit is contained in:
parent
e28858bb29
commit
86ccdcc9d9
8 changed files with 538 additions and 149 deletions
|
@ -30,6 +30,14 @@ impl<'a> Cursor<'a> {
|
|||
self.chars.clone().next().unwrap_or(EOF_CHAR)
|
||||
}
|
||||
|
||||
/// Peeks the second character from the input stream without consuming it.
|
||||
/// Returns [`EOF_CHAR`] if the position is past the end of the file.
|
||||
pub fn second(&self) -> char {
|
||||
let mut chars = self.chars.clone();
|
||||
chars.next();
|
||||
chars.next().unwrap_or(EOF_CHAR)
|
||||
}
|
||||
|
||||
/// Peeks the next character from the input stream without consuming it.
|
||||
/// Returns [`EOF_CHAR`] if the file is at the end of the file.
|
||||
pub fn last(&self) -> char {
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue