mirror of
https://github.com/astral-sh/ruff.git
synced 2025-08-16 16:40:36 +00:00
Refactor binary expression parsing (#11073)
## Summary This PR refactors the binary expression parsing in a way to make it readable and easy to understand. It draws inspiration from the suggested edits in the linked messages in #10752. ### Changes * Ability to get the precedence of an operator * From a boolean operator (`BinOp`) to `OperatorPrecedence` * From a binary operator (`Operator`) to `OperatorPrecedence` * No comparison operator because all of them have the same precedence * Implement methods on `TokenKind` to convert it to an appropriate operator enum * Add `as_boolean_operator` which returns an `Option<BoolOp>` * Add `as_binary_operator` which returns an `Option<Operator>` * No `as_comparison_operator` because it requires lookahead and I'm not sure if `token.as_comparison_operator(peek)` is a good way to implement it * Introduce `BinaryLikeOperator` * Constructed from two tokens using the methods from the second point * Add `precedence` method using the conversion methods mentioned in the first point * Make most of the functions in `TokenKind` private to the module * Use `self` instead of `&self` for `TokenKind` fixes: #11072 ## Test Plan Refer #11088
This commit is contained in:
Dhruv Manilawala
GitHub
parent
5b81b8368d
commit
7eba967e16
3 changed files with 222 additions and 204 deletions
|
|
@ -536,17 +536,17 @@ pub enum TokenKind {
|
|||
|
||||
impl TokenKind {
|
||||
#[inline]
|
||||
pub const fn is_newline(&self) -> bool {
|
||||
pub const fn is_newline(self) -> bool {
|
||||
matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_unary(&self) -> bool {
|
||||
pub const fn is_unary(self) -> bool {
|
||||
matches!(self, TokenKind::Plus | TokenKind::Minus)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_keyword(&self) -> bool {
|
||||
pub const fn is_keyword(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
TokenKind::False
|
||||
|
|
@ -587,7 +587,7 @@ impl TokenKind {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_operator(&self) -> bool {
|
||||
pub const fn is_operator(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
TokenKind::Lpar
|
||||
|
|
@ -646,7 +646,7 @@ impl TokenKind {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_singleton(&self) -> bool {
|
||||
pub const fn is_singleton(self) -> bool {
|
||||
matches!(self, TokenKind::False | TokenKind::True | TokenKind::None)
|
||||
}
|
||||
|
||||
|
|
@ -663,7 +663,7 @@ impl TokenKind {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_arithmetic(&self) -> bool {
|
||||
pub const fn is_arithmetic(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
TokenKind::DoubleStar
|
||||
|
|
@ -677,7 +677,7 @@ impl TokenKind {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_bitwise_or_shift(&self) -> bool {
|
||||
pub const fn is_bitwise_or_shift(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
TokenKind::LeftShift
|
||||
|
|
@ -695,51 +695,65 @@ impl TokenKind {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_soft_keyword(&self) -> bool {
|
||||
pub const fn is_soft_keyword(self) -> bool {
|
||||
matches!(self, TokenKind::Match | TokenKind::Case)
|
||||
}
|
||||
|
||||
/// Returns the [`BoolOp`] that corresponds to this token kind, if it is a boolean operator,
|
||||
/// otherwise return [None].
|
||||
#[inline]
|
||||
pub const fn is_compare_operator(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
TokenKind::Not
|
||||
| TokenKind::In
|
||||
| TokenKind::Is
|
||||
| TokenKind::EqEqual
|
||||
| TokenKind::NotEqual
|
||||
| TokenKind::Less
|
||||
| TokenKind::LessEqual
|
||||
| TokenKind::Greater
|
||||
| TokenKind::GreaterEqual
|
||||
)
|
||||
pub(crate) const fn as_bool_operator(self) -> Option<BoolOp> {
|
||||
Some(match self {
|
||||
TokenKind::And => BoolOp::And,
|
||||
TokenKind::Or => BoolOp::Or,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub const fn is_bool_operator(&self) -> bool {
|
||||
matches!(self, TokenKind::And | TokenKind::Or)
|
||||
/// Returns the binary [`Operator`] that corresponds to the current token, if it's a binary
|
||||
/// operator, otherwise return [None].
|
||||
///
|
||||
/// Use [`TokenKind::as_augmented_assign_operator`] to match against an augmented assignment
|
||||
/// token.
|
||||
pub(crate) const fn as_binary_operator(self) -> Option<Operator> {
|
||||
Some(match self {
|
||||
TokenKind::Plus => Operator::Add,
|
||||
TokenKind::Minus => Operator::Sub,
|
||||
TokenKind::Star => Operator::Mult,
|
||||
TokenKind::At => Operator::MatMult,
|
||||
TokenKind::DoubleStar => Operator::Pow,
|
||||
TokenKind::Slash => Operator::Div,
|
||||
TokenKind::DoubleSlash => Operator::FloorDiv,
|
||||
TokenKind::Percent => Operator::Mod,
|
||||
TokenKind::Amper => Operator::BitAnd,
|
||||
TokenKind::Vbar => Operator::BitOr,
|
||||
TokenKind::CircumFlex => Operator::BitXor,
|
||||
TokenKind::LeftShift => Operator::LShift,
|
||||
TokenKind::RightShift => Operator::RShift,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the [`Operator`] that corresponds to this token kind, if it is
|
||||
/// an augmented assignment operator, or [`None`] otherwise.
|
||||
#[inline]
|
||||
pub const fn as_augmented_assign_operator(&self) -> Option<Operator> {
|
||||
match self {
|
||||
TokenKind::PlusEqual => Some(Operator::Add),
|
||||
TokenKind::MinusEqual => Some(Operator::Sub),
|
||||
TokenKind::StarEqual => Some(Operator::Mult),
|
||||
TokenKind::AtEqual => Some(Operator::MatMult),
|
||||
TokenKind::DoubleStarEqual => Some(Operator::Pow),
|
||||
TokenKind::SlashEqual => Some(Operator::Div),
|
||||
TokenKind::DoubleSlashEqual => Some(Operator::FloorDiv),
|
||||
TokenKind::PercentEqual => Some(Operator::Mod),
|
||||
TokenKind::AmperEqual => Some(Operator::BitAnd),
|
||||
TokenKind::VbarEqual => Some(Operator::BitOr),
|
||||
TokenKind::CircumflexEqual => Some(Operator::BitXor),
|
||||
TokenKind::LeftShiftEqual => Some(Operator::LShift),
|
||||
TokenKind::RightShiftEqual => Some(Operator::RShift),
|
||||
_ => None,
|
||||
}
|
||||
pub(crate) const fn as_augmented_assign_operator(self) -> Option<Operator> {
|
||||
Some(match self {
|
||||
TokenKind::PlusEqual => Operator::Add,
|
||||
TokenKind::MinusEqual => Operator::Sub,
|
||||
TokenKind::StarEqual => Operator::Mult,
|
||||
TokenKind::AtEqual => Operator::MatMult,
|
||||
TokenKind::DoubleStarEqual => Operator::Pow,
|
||||
TokenKind::SlashEqual => Operator::Div,
|
||||
TokenKind::DoubleSlashEqual => Operator::FloorDiv,
|
||||
TokenKind::PercentEqual => Operator::Mod,
|
||||
TokenKind::AmperEqual => Operator::BitAnd,
|
||||
TokenKind::VbarEqual => Operator::BitOr,
|
||||
TokenKind::CircumflexEqual => Operator::BitXor,
|
||||
TokenKind::LeftShiftEqual => Operator::LShift,
|
||||
TokenKind::RightShiftEqual => Operator::RShift,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
pub const fn from_token(token: &Tok) -> Self {
|
||||
|
|
@ -865,41 +879,6 @@ impl From<Tok> for TokenKind {
|
|||
}
|
||||
}
|
||||
|
||||
impl TryFrom<TokenKind> for Operator {
|
||||
type Error = ();
|
||||
|
||||
fn try_from(value: TokenKind) -> Result<Self, Self::Error> {
|
||||
Ok(match value {
|
||||
TokenKind::At | TokenKind::AtEqual => Operator::MatMult,
|
||||
TokenKind::Plus | TokenKind::PlusEqual => Operator::Add,
|
||||
TokenKind::Star | TokenKind::StarEqual => Operator::Mult,
|
||||
TokenKind::Vbar | TokenKind::VbarEqual => Operator::BitOr,
|
||||
TokenKind::Minus | TokenKind::MinusEqual => Operator::Sub,
|
||||
TokenKind::Slash | TokenKind::SlashEqual => Operator::Div,
|
||||
TokenKind::Amper | TokenKind::AmperEqual => Operator::BitAnd,
|
||||
TokenKind::Percent | TokenKind::PercentEqual => Operator::Mod,
|
||||
TokenKind::DoubleStar | TokenKind::DoubleStarEqual => Operator::Pow,
|
||||
TokenKind::LeftShift | TokenKind::LeftShiftEqual => Operator::LShift,
|
||||
TokenKind::CircumFlex | TokenKind::CircumflexEqual => Operator::BitXor,
|
||||
TokenKind::RightShift | TokenKind::RightShiftEqual => Operator::RShift,
|
||||
TokenKind::DoubleSlash | TokenKind::DoubleSlashEqual => Operator::FloorDiv,
|
||||
_ => return Err(()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<TokenKind> for BoolOp {
|
||||
type Error = ();
|
||||
|
||||
fn try_from(value: TokenKind) -> Result<Self, Self::Error> {
|
||||
Ok(match value {
|
||||
TokenKind::And => BoolOp::And,
|
||||
TokenKind::Or => BoolOp::Or,
|
||||
_ => return Err(()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<&Tok> for UnaryOp {
|
||||
type Error = String;
|
||||
|
||||
|
|
@ -922,6 +901,37 @@ impl TryFrom<TokenKind> for UnaryOp {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<BoolOp> for TokenKind {
|
||||
#[inline]
|
||||
fn from(op: BoolOp) -> Self {
|
||||
match op {
|
||||
BoolOp::And => TokenKind::And,
|
||||
BoolOp::Or => TokenKind::Or,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Operator> for TokenKind {
|
||||
#[inline]
|
||||
fn from(op: Operator) -> Self {
|
||||
match op {
|
||||
Operator::Add => TokenKind::Plus,
|
||||
Operator::Sub => TokenKind::Minus,
|
||||
Operator::Mult => TokenKind::Star,
|
||||
Operator::MatMult => TokenKind::At,
|
||||
Operator::Div => TokenKind::Slash,
|
||||
Operator::Mod => TokenKind::Percent,
|
||||
Operator::Pow => TokenKind::DoubleStar,
|
||||
Operator::LShift => TokenKind::LeftShift,
|
||||
Operator::RShift => TokenKind::RightShift,
|
||||
Operator::BitOr => TokenKind::Vbar,
|
||||
Operator::BitXor => TokenKind::CircumFlex,
|
||||
Operator::BitAnd => TokenKind::Amper,
|
||||
Operator::FloorDiv => TokenKind::DoubleSlash,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for TokenKind {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let value = match self {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue