language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-02 18:02:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Remove leftover marker tokens (#11444)

Browse source 
## Summary

This PR removes the leftover marker tokens from the LALRPOP to
hand-written parser migration.
This commit is contained in:
Dhruv Manilawala 2024-05-16 17:09:05 +05:30 • committed by GitHub
parent 4436dec1d9
commit f67c02c837
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 7 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

32
crates/ruff_python_parser/src/token.rs
View file

@ -5,10 +5,9 @@
//! //!
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens //! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Grammar/Tokens
use ruff_python_ast::{AnyStringFlags, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};
use std::fmt; use std::fmt;
use crate::Mode; use ruff_python_ast::{AnyStringFlags, BoolOp, Int, IpyEscapeKind, Operator, UnaryOp};
/// The set of tokens the Python source code can be tokenized in. /// The set of tokens the Python source code can be tokenized in.
#[derive(Clone, Debug, PartialEq, is_macro::Is)] #[derive(Clone, Debug, PartialEq, is_macro::Is)]
@ -61,7 +60,9 @@ pub enum Tok {
/// Token value for the end of an f-string. This includes the closing quote. /// Token value for the end of an f-string. This includes the closing quote.
FStringEnd, FStringEnd,
/// Token value for IPython escape commands. These are recognized by the lexer /// Token value for IPython escape commands. These are recognized by the lexer
/// only when the mode is [`Mode::Ipython`]. /// only when the mode is [`Ipython`].
///
/// [`Ipython`]: crate::Mode::Ipython
IpyEscapeCommand { IpyEscapeCommand {
/// The magic command value. /// The magic command value.
value: Box<str>, value: Box<str>,
@ -80,7 +81,9 @@ pub enum Tok {
/// Token value for a dedent. /// Token value for a dedent.
Dedent, Dedent,
EndOfFile, EndOfFile,
/// Token value for a question mark `?`. This is only used in [`Mode::Ipython`]. /// Token value for a question mark `?`. This is only used in [`Ipython`].
///
/// [`Ipython`]: crate::Mode::Ipython
Question, Question,
/// Token value for a exclamation mark `!`. /// Token value for a exclamation mark `!`.
Exclamation, Exclamation,
@ -222,9 +225,6 @@ pub enum Tok {
Yield, Yield,
Unknown, Unknown,
// RustPython specific.
StartModule,
StartExpression,
} }
impl Tok { impl Tok {
@ -232,13 +232,6 @@ impl Tok {
pub fn kind(&self) -> TokenKind { pub fn kind(&self) -> TokenKind {
TokenKind::from_token(self) TokenKind::from_token(self)
} }
pub fn start_marker(mode: Mode) -> Self {
match mode {
Mode::Module | Mode::Ipython => Tok::StartModule,
Mode::Expression => Tok::StartExpression,
}
}
} }
impl fmt::Display for Tok { impl fmt::Display for Tok {
@ -261,8 +254,6 @@ impl fmt::Display for Tok {
NonLogicalNewline => f.write_str("NonLogicalNewline"), NonLogicalNewline => f.write_str("NonLogicalNewline"),
Indent => f.write_str("Indent"), Indent => f.write_str("Indent"),
Dedent => f.write_str("Dedent"), Dedent => f.write_str("Dedent"),
StartModule => f.write_str("StartProgram"),
StartExpression => f.write_str("StartExpression"),
EndOfFile => f.write_str("EOF"), EndOfFile => f.write_str("EOF"),
Question => f.write_str("?"), Question => f.write_str("?"),
Exclamation => f.write_str("!"), Exclamation => f.write_str("!"),
@ -537,10 +528,6 @@ pub enum TokenKind {
Yield, Yield,
Unknown, Unknown,
// RustPython specific.
StartModule,
StartInteractive,
StartExpression,
} }
impl TokenKind { impl TokenKind {
@ -900,8 +887,6 @@ impl TokenKind {
Tok::With => TokenKind::With, Tok::With => TokenKind::With,
Tok::Yield => TokenKind::Yield, Tok::Yield => TokenKind::Yield,
Tok::Unknown => TokenKind::Unknown, Tok::Unknown => TokenKind::Unknown,
Tok::StartModule => TokenKind::StartModule,
Tok::StartExpression => TokenKind::StartExpression,
} }
} }
} }
@ -965,9 +950,6 @@ impl fmt::Display for TokenKind {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let value = match self { let value = match self {
TokenKind::Unknown => "Unknown", TokenKind::Unknown => "Unknown",
TokenKind::StartModule => "StartModule",
TokenKind::StartExpression => "StartExpression",
TokenKind::StartInteractive => "StartInteractive",
TokenKind::Newline => "newline", TokenKind::Newline => "newline",
TokenKind::NonLogicalNewline => "NonLogicalNewline", TokenKind::NonLogicalNewline => "NonLogicalNewline",
TokenKind::Indent => "indent", TokenKind::Indent => "indent",