language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-16 09:35:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Include soft keywords for is_keyword check (#11445)

Browse source 
## Summary

This PR updates the `TokenKind::is_keyword` check to include soft
keywords. To account for this change, it adds a new
`is_non_soft_keyword` method.

The usage in logical line rules were updated to use the
`is_non_soft_keyword` method but it'll be updated to use `is_keyword` in
a follow-up PR (#11446).

While, the parser usages were kept as is. And because of that, the
snapshots for two test cases were updated in a better direction.

## Test Plan

`cargo insta test`
This commit is contained in:
Dhruv Manilawala 2024-05-17 10:26:48 +05:30 • committed by GitHub
parent 43e8147eaf
commit 83152fff92
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 94 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_after_keyword.rs
View file

@ -52,7 +52,7 @@ pub(crate) fn missing_whitespace_after_keyword(
let tok0_kind = tok0.kind(); let tok0_kind = tok0.kind();
let tok1_kind = tok1.kind(); let tok1_kind = tok1.kind();
if tok0_kind.is_keyword() if tok0_kind.is_non_soft_keyword()
&& !(tok0_kind.is_singleton() && !(tok0_kind.is_singleton()
|| matches!(tok0_kind, TokenKind::Async | TokenKind::Await) || matches!(tok0_kind, TokenKind::Async | TokenKind::Await)
|| tok0_kind == TokenKind::Except && tok1_kind == TokenKind::Star || tok0_kind == TokenKind::Except && tok1_kind == TokenKind::Star

4
crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/missing_whitespace_around_operator.rs
View file

@ -198,9 +198,7 @@ pub(crate) fn missing_whitespace_around_operator(
matches!( matches!(
prev_kind, prev_kind,
TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace TokenKind::Rpar | TokenKind::Rsqb | TokenKind::Rbrace
) || !(prev_kind.is_operator() ) || !(prev_kind.is_operator() || prev_kind.is_keyword())
|| prev_kind.is_keyword()
|| prev_kind.is_soft_keyword())
}; };
if is_binary { if is_binary {

2
crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/mod.rs
View file

@ -445,7 +445,7 @@ impl LogicalLinesBuilder {
if matches!(kind, TokenKind::Comma | TokenKind::Semi | TokenKind::Colon) { if matches!(kind, TokenKind::Comma | TokenKind::Semi | TokenKind::Colon) {
line.flags.insert(TokenFlags::PUNCTUATION); line.flags.insert(TokenFlags::PUNCTUATION);
} else if kind.is_keyword() { } else if kind.is_non_soft_keyword() {
line.flags.insert(TokenFlags::KEYWORD); line.flags.insert(TokenFlags::KEYWORD);
} }

6
crates/ruff_linter/src/rules/pycodestyle/rules/logical_lines/whitespace_around_keywords.rs
View file

@ -127,8 +127,8 @@ pub(crate) fn whitespace_around_keywords(line: &LogicalLine, context: &mut Logic
let mut after_keyword = false; let mut after_keyword = false;
for token in line.tokens() { for token in line.tokens() {
let is_keyword = token.kind().is_keyword(); let is_non_soft_keyword = token.kind().is_non_soft_keyword();
if is_keyword { if is_non_soft_keyword {
if !after_keyword { if !after_keyword {
match line.leading_whitespace(token) { match line.leading_whitespace(token) {
(Whitespace::Tab, offset) => { (Whitespace::Tab, offset) => {
@ -184,6 +184,6 @@ pub(crate) fn whitespace_around_keywords(line: &LogicalLine, context: &mut Logic
} }
} }
after_keyword = is_keyword; after_keyword = is_non_soft_keyword;
} }
} }

85
crates/ruff_python_parser/src/token.rs
View file

@ -352,7 +352,7 @@ impl fmt::Display for Tok {
/// ///
/// This is a lightweight representation of [`Tok`] which doesn't contain any information /// This is a lightweight representation of [`Tok`] which doesn't contain any information
/// about the token itself. /// about the token itself.
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
pub enum TokenKind { pub enum TokenKind {
/// Token value for a name, commonly known as an identifier. /// Token value for a name, commonly known as an identifier.
Name, Name,
@ -485,12 +485,10 @@ pub enum TokenKind {
/// Token value for ellipsis `...`. /// Token value for ellipsis `...`.
Ellipsis, Ellipsis,
// Self documenting. // The keywords should be sorted in alphabetical order. If the boundary tokens for the
// Keywords (alphabetically): // "Keywords" and "Soft keywords" group change, update the related methods on `TokenKind`.
False,
None,
True,
// Keywords
And, And,
As, As,
Assert, Assert,
@ -504,6 +502,7 @@ pub enum TokenKind {
Elif, Elif,
Else, Else,
Except, Except,
False,
Finally, Finally,
For, For,
From, From,
@ -513,20 +512,24 @@ pub enum TokenKind {
In, In,
Is, Is,
Lambda, Lambda,
None,
Nonlocal, Nonlocal,
Not, Not,
Or, Or,
Pass, Pass,
Raise, Raise,
Return, Return,
True,
Try, Try,
While, While,
Match,
Type,
Case,
With, With,
Yield, Yield,
// Soft keywords
Case,
Match,
Type,
Unknown, Unknown,
} }
@ -536,45 +539,28 @@ impl TokenKind {
matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline) matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline)
} }
/// Returns `true` if the token is a keyword (including soft keywords).
///
/// See also [`TokenKind::is_soft_keyword`], [`TokenKind::is_non_soft_keyword`].
#[inline] #[inline]
pub const fn is_keyword(self) -> bool { pub fn is_keyword(self) -> bool {
matches!( TokenKind::And <= self && self <= TokenKind::Type
self, }
TokenKind::False
| TokenKind::True /// Returns `true` if the token is strictly a soft keyword.
| TokenKind::None ///
| TokenKind::And /// See also [`TokenKind::is_keyword`], [`TokenKind::is_non_soft_keyword`].
| TokenKind::As #[inline]
| TokenKind::Assert pub fn is_soft_keyword(self) -> bool {
| TokenKind::Await TokenKind::Case <= self && self <= TokenKind::Type
| TokenKind::Break }
| TokenKind::Class
| TokenKind::Continue /// Returns `true` if the token is strictly a non-soft keyword.
| TokenKind::Def ///
| TokenKind::Del /// See also [`TokenKind::is_keyword`], [`TokenKind::is_soft_keyword`].
| TokenKind::Elif #[inline]
| TokenKind::Else pub fn is_non_soft_keyword(self) -> bool {
| TokenKind::Except TokenKind::And <= self && self <= TokenKind::Yield
| TokenKind::Finally
| TokenKind::For
| TokenKind::From
| TokenKind::Global
| TokenKind::If
| TokenKind::Import
| TokenKind::In
| TokenKind::Is
| TokenKind::Lambda
| TokenKind::Nonlocal
| TokenKind::Not
| TokenKind::Or
| TokenKind::Pass
| TokenKind::Raise
| TokenKind::Return
| TokenKind::Try
| TokenKind::While
| TokenKind::With
| TokenKind::Yield
)
} }
#[inline] #[inline]
@ -685,11 +671,6 @@ impl TokenKind {
) )
} }
#[inline]
pub const fn is_soft_keyword(self) -> bool {
matches!(self, TokenKind::Match | TokenKind::Case)
}
/// Returns `true` if the current token is a unary arithmetic operator. /// Returns `true` if the current token is a unary arithmetic operator.
#[inline] #[inline]
pub const fn is_unary_arithmetic_operator(self) -> bool { pub const fn is_unary_arithmetic_operator(self) -> bool {

82
crates/ruff_python_parser/tests/snapshots/invalid_syntax@ann_assign_stmt_type_alias_annotation.py.snap
View file

@ -11,7 +11,7 @@ Module(
body: [ body: [
AnnAssign( AnnAssign(
StmtAnnAssign { StmtAnnAssign {
range: 0..2, range: 0..7,
target: Name( target: Name(
ExprName { ExprName {
range: 0..1, range: 0..1,
@ -21,26 +21,27 @@ Module(
), ),
annotation: Name( annotation: Name(
ExprName { ExprName {
range: 2..2, range: 3..7,
id: "", id: "type",
ctx: Invalid, ctx: Load,
}, },
), ),
value: None, value: None,
simple: true, simple: true,
}, },
), ),
TypeAlias( Assign(
StmtTypeAlias { StmtAssign {
range: 3..15, range: 8..15,
name: Name( targets: [
ExprName { Name(
range: 8..9, ExprName {
id: "X", range: 8..9,
ctx: Store, id: "X",
}, ctx: Store,
), },
type_params: None, ),
],
value: Name( value: Name(
ExprName { ExprName {
range: 12..15, range: 12..15,
@ -52,33 +53,34 @@ Module(
), ),
Expr( Expr(
StmtExpr { StmtExpr {
range: 16..23, range: 16..28,
value: Lambda( value: Lambda(
ExprLambda { ExprLambda {
range: 16..23, range: 16..28,
parameters: None, parameters: None,
body: Name( body: Name(
ExprName { ExprName {
range: 23..23, range: 24..28,
id: "", id: "type",
ctx: Invalid, ctx: Load,
}, },
), ),
}, },
), ),
}, },
), ),
TypeAlias( Assign(
StmtTypeAlias { StmtAssign {
range: 24..36, range: 29..36,
name: Name( targets: [
ExprName { Name(
range: 29..30, ExprName {
id: "X", range: 29..30,
ctx: Store, id: "X",
}, ctx: Store,
), },
type_params: None, ),
],
value: Name( value: Name(
ExprName { ExprName {
range: 33..36, range: 33..36,
@ -96,7 +98,14 @@ Module(
| |
1 | a: type X = int 1 | a: type X = int
| ^^^^ Syntax Error: Expected an expression | ^^^^ Syntax Error: Expected an identifier, but found a keyword 'type' that cannot be used here
2 | lambda: type X = int
|
|
1 | a: type X = int
| ^ Syntax Error: Simple statements must be separated by newlines or semicolons
2 | lambda: type X = int 2 | lambda: type X = int
| |
@ -104,5 +113,12 @@ Module(
| |
1 | a: type X = int 1 | a: type X = int
2 | lambda: type X = int 2 | lambda: type X = int
| ^^^^ Syntax Error: Expected an expression | ^^^^ Syntax Error: Expected an identifier, but found a keyword 'type' that cannot be used here
|
|
1 | a: type X = int
2 | lambda: type X = int
| ^ Syntax Error: Simple statements must be separated by newlines or semicolons
| |

46
crates/ruff_python_parser/tests/snapshots/invalid_syntax@comprehension_missing_for_after_async.py.snap
View file

@ -12,38 +12,11 @@ Module(
Expr( Expr(
StmtExpr { StmtExpr {
range: 0..7, range: 0..7,
value: Generator( value: Name(
ExprGenerator { ExprName {
range: 0..7, range: 1..6,
elt: Name( id: "async",
ExprName { ctx: Load,
range: 1..1,
id: "",
ctx: Invalid,
},
),
generators: [
Comprehension {
range: 1..6,
target: Name(
ExprName {
range: 6..6,
id: "",
ctx: Store,
},
),
iter: Name(
ExprName {
range: 6..6,
id: "",
ctx: Invalid,
},
),
ifs: [],
is_async: true,
},
],
parenthesized: true,
}, },
), ),
}, },
@ -95,14 +68,7 @@ Module(
| |
1 | (async) 1 | (async)
| ^^^^^ Syntax Error: Expected an expression | ^^^^^ Syntax Error: Expected an identifier, but found a keyword 'async' that cannot be used here
2 | (x async x in iter)
|
|
1 | (async)
| ^ Syntax Error: Expected 'for', found ')'
2 | (x async x in iter) 2 | (x async x in iter)
| |