mirror of
https://github.com/astral-sh/ruff.git
synced 2025-08-16 08:30:30 +00:00
Add Tokens
newtype wrapper, TokenKind
iterator (#11361)
## Summary Alternative to #11237 This PR adds a new `Tokens` struct which is a newtype wrapper around a vector of lexer output. This allows us to add a `kinds` method which returns an iterator over the corresponding `TokenKind`. This iterator is implemented as a separate `TokenKindIter` struct to allow using the type and provide additional methods like `peek` directly on the iterator. This exposes the linter to access the stream of `TokenKind` instead of `Tok`. Edit: I've made the necessary downstream changes and plan to merge the entire stack at once.
This commit is contained in:
parent
50f14d017e
commit
025768d303
9 changed files with 142 additions and 25 deletions
|
@ -228,6 +228,11 @@ pub enum Tok {
|
|||
}
|
||||
|
||||
impl Tok {
|
||||
#[inline]
|
||||
pub fn kind(&self) -> TokenKind {
|
||||
TokenKind::from_token(self)
|
||||
}
|
||||
|
||||
pub fn start_marker(mode: Mode) -> Self {
|
||||
match mode {
|
||||
Mode::Module | Mode::Ipython => Tok::StartModule,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue