mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-23 13:05:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Avoid match statement misidentification in token rules (#3129)

Browse source
This commit is contained in:
Charlie Marsh 2023-02-22 10:44:45 -05:00 • committed by GitHub
parent df3932f750
commit 1efa2e07ad
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 128 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

8
Cargo.lock generated
View file

@ -2150,7 +2150,7 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython.git?rev=ddf497623ae56d21aa4166ff1c0725a7db67e955#ddf497623ae56d21aa4166ff1c0725a7db67e955"
source = "git+https://github.com/RustPython/RustPython.git?rev=6d5bbd913c7c46518f4ed8b1b378ffb0df72f505#6d5bbd913c7c46518f4ed8b1b378ffb0df72f505"
dependencies = [
"num-bigint",
"rustpython-compiler-core",
@ -2159,7 +2159,7 @@ dependencies = [
[[package]]
name = "rustpython-common"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython.git?rev=ddf497623ae56d21aa4166ff1c0725a7db67e955#ddf497623ae56d21aa4166ff1c0725a7db67e955"
source = "git+https://github.com/RustPython/RustPython.git?rev=6d5bbd913c7c46518f4ed8b1b378ffb0df72f505#6d5bbd913c7c46518f4ed8b1b378ffb0df72f505"
dependencies = [
"ascii",
"bitflags",
@ -2184,7 +2184,7 @@ dependencies = [
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython.git?rev=ddf497623ae56d21aa4166ff1c0725a7db67e955#ddf497623ae56d21aa4166ff1c0725a7db67e955"
source = "git+https://github.com/RustPython/RustPython.git?rev=6d5bbd913c7c46518f4ed8b1b378ffb0df72f505#6d5bbd913c7c46518f4ed8b1b378ffb0df72f505"
dependencies = [
"bincode",
"bitflags",
@ -2201,7 +2201,7 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.2.0"
source = "git+https://github.com/RustPython/RustPython.git?rev=ddf497623ae56d21aa4166ff1c0725a7db67e955#ddf497623ae56d21aa4166ff1c0725a7db67e955"
source = "git+https://github.com/RustPython/RustPython.git?rev=6d5bbd913c7c46518f4ed8b1b378ffb0df72f505#6d5bbd913c7c46518f4ed8b1b378ffb0df72f505"
dependencies = [
"ahash",
"anyhow",

4
Cargo.toml
View file

@ -13,8 +13,8 @@ libcst = { git = "https://github.com/charliermarsh/LibCST", rev = "f2f0b7a487a87
once_cell = { version = "1.16.0" }
regex = { version = "1.6.0" }
rustc-hash = { version = "1.1.0" }
rustpython-common = { git = "https://github.com/RustPython/RustPython.git", rev = "ddf497623ae56d21aa4166ff1c0725a7db67e955" }
rustpython-parser = { features = ["lalrpop"], git = "https://github.com/RustPython/RustPython.git", rev = "ddf497623ae56d21aa4166ff1c0725a7db67e955" }
rustpython-common = { git = "https://github.com/RustPython/RustPython.git", rev = "6d5bbd913c7c46518f4ed8b1b378ffb0df72f505" }
rustpython-parser = { features = ["lalrpop"], git = "https://github.com/RustPython/RustPython.git", rev = "6d5bbd913c7c46518f4ed8b1b378ffb0df72f505" }
schemars = { version = "0.8.11" }
serde = { version = "1.0.147", features = ["derive"] }
serde_json = { version = "1.0.87" }

3
crates/ruff/resources/test/fixtures/pycodestyle/E70.py vendored
View file

@ -57,3 +57,6 @@ class C: ...; ...
#: E701:2:12
match *0, 1, *2:
case 0,: y = 0
#:
class Foo:
match: Optional[Match] = None

24
crates/ruff/src/ast/helpers.rs
View file

@ -11,6 +11,7 @@ use rustpython_parser::ast::{
};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use rustpython_parser::token::StringKind;
use smallvec::{smallvec, SmallVec};
@ -655,7 +656,7 @@ pub fn has_comments<T>(located: &Located<T>, locator: &Locator) -> bool {
/// Returns `true` if a [`Range`] includes at least one comment.
pub fn has_comments_in(range: Range, locator: &Locator) -> bool {
for tok in lexer::make_tokenizer(locator.slice(&range)) {
for tok in lexer::make_tokenizer_located(locator.slice(&range), Mode::Module, range.location) {
match tok {
Ok((_, tok, _)) => {
if matches!(tok, Tok::Comment(..)) {
@ -870,7 +871,8 @@ pub fn match_parens(start: Location, locator: &Locator) -> Option<Range> {
let mut fix_start = None;
let mut fix_end = None;
let mut count: usize = 0;
for (start, tok, end) in lexer::make_tokenizer_located(contents, start).flatten() {
for (start, tok, end) in lexer::make_tokenizer_located(contents, Mode::Module, start).flatten()
{
if matches!(tok, Tok::Lpar) {
if count == 0 {
fix_start = Some(start);
@ -902,7 +904,9 @@ pub fn identifier_range(stmt: &Stmt, locator: &Locator) -> Range {
| StmtKind::AsyncFunctionDef { .. }
) {
let contents = locator.slice(&Range::from_located(stmt));
for (start, tok, end) in lexer::make_tokenizer_located(contents, stmt.location).flatten() {
for (start, tok, end) in
lexer::make_tokenizer_located(contents, Mode::Module, stmt.location).flatten()
{
if matches!(tok, Tok::Name { .. }) {
return Range::new(start, end);
}
@ -933,7 +937,7 @@ pub fn find_names<'a, T, U>(
locator: &'a Locator,
) -> impl Iterator<Item = Range> + 'a {
let contents = locator.slice(&Range::from_located(located));
lexer::make_tokenizer_located(contents, located.location)
lexer::make_tokenizer_located(contents, Mode::Module, located.location)
.flatten()
.filter(|(_, tok, _)| matches!(tok, Tok::Name { .. }))
.map(|(start, _, end)| Range {
@ -951,7 +955,7 @@ pub fn excepthandler_name_range(handler: &Excepthandler, locator: &Locator) -> O
(Some(_), Some(type_)) => {
let type_end_location = type_.end_location.unwrap();
let contents = locator.slice(&Range::new(type_end_location, body[0].location));
let range = lexer::make_tokenizer_located(contents, type_end_location)
let range = lexer::make_tokenizer_located(contents, Mode::Module, type_end_location)
.flatten()
.tuple_windows()
.find(|(tok, next_tok)| {
@ -978,7 +982,7 @@ pub fn except_range(handler: &Excepthandler, locator: &Locator) -> Range {
location: handler.location,
end_location: end,
});
let range = lexer::make_tokenizer_located(contents, handler.location)
let range = lexer::make_tokenizer_located(contents, Mode::Module, handler.location)
.flatten()
.find(|(_, kind, _)| matches!(kind, Tok::Except { .. }))
.map(|(location, _, end_location)| Range {
@ -992,7 +996,7 @@ pub fn except_range(handler: &Excepthandler, locator: &Locator) -> Range {
/// Find f-strings that don't contain any formatted values in a `JoinedStr`.
pub fn find_useless_f_strings(expr: &Expr, locator: &Locator) -> Vec<(Range, Range)> {
let contents = locator.slice(&Range::from_located(expr));
lexer::make_tokenizer_located(contents, expr.location)
lexer::make_tokenizer_located(contents, Mode::Module, expr.location)
.flatten()
.filter_map(|(location, tok, end_location)| match tok {
Tok::String {
@ -1046,7 +1050,7 @@ pub fn else_range(stmt: &Stmt, locator: &Locator) -> Option<Range> {
.expect("Expected orelse to be non-empty")
.location,
});
let range = lexer::make_tokenizer_located(contents, body_end)
let range = lexer::make_tokenizer_located(contents, Mode::Module, body_end)
.flatten()
.find(|(_, kind, _)| matches!(kind, Tok::Else))
.map(|(location, _, end_location)| Range {
@ -1062,7 +1066,7 @@ pub fn else_range(stmt: &Stmt, locator: &Locator) -> Option<Range> {
/// Return the `Range` of the first `Tok::Colon` token in a `Range`.
pub fn first_colon_range(range: Range, locator: &Locator) -> Option<Range> {
let contents = locator.slice(&range);
let range = lexer::make_tokenizer_located(contents, range.location)
let range = lexer::make_tokenizer_located(contents, Mode::Module, range.location)
.flatten()
.find(|(_, kind, _)| matches!(kind, Tok::Colon))
.map(|(location, _, end_location)| Range {
@ -1092,7 +1096,7 @@ pub fn elif_else_range(stmt: &Stmt, locator: &Locator) -> Option<Range> {
_ => return None,
};
let contents = locator.slice(&Range::new(start, end));
let range = lexer::make_tokenizer_located(contents, start)
let range = lexer::make_tokenizer_located(contents, Mode::Module, start)
.flatten()
.find(|(_, kind, _)| matches!(kind, Tok::Elif | Tok::Else))
.map(|(location, _, end_location)| Range {

5
crates/ruff/src/ast/operations.rs
View file

@ -3,6 +3,7 @@ use rustc_hash::FxHashMap;
use rustpython_parser::ast::{Cmpop, Constant, Expr, ExprKind, Located, Stmt, StmtKind};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use crate::ast::helpers::any_over_expr;
use crate::ast::types::{BindingKind, Scope};
@ -283,7 +284,9 @@ pub type LocatedCmpop<U = ()> = Located<Cmpop, U>;
/// `CPython` doesn't either. This method iterates over the token stream and
/// re-identifies [`Cmpop`] nodes, annotating them with valid ranges.
pub fn locate_cmpops(contents: &str) -> Vec<LocatedCmpop> {
let mut tok_iter = lexer::make_tokenizer(contents).flatten().peekable();
let mut tok_iter = lexer::make_tokenizer(contents, Mode::Module)
.flatten()
.peekable();
let mut ops: Vec<LocatedCmpop> = vec![];
let mut count: usize = 0;
loop {

13
crates/ruff/src/autofix/helpers.rs
View file

@ -6,6 +6,7 @@ use libcst_native::{
use rustpython_parser::ast::{ExcepthandlerKind, Expr, Keyword, Location, Stmt, StmtKind};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use crate::ast::helpers;
use crate::ast::helpers::to_absolute;
@ -371,7 +372,9 @@ pub fn remove_argument(
if n_arguments == 1 {
// Case 1: there is only one argument.
let mut count: usize = 0;
for (start, tok, end) in lexer::make_tokenizer_located(contents, stmt_at).flatten() {
for (start, tok, end) in
lexer::make_tokenizer_located(contents, Mode::Module, stmt_at).flatten()
{
if matches!(tok, Tok::Lpar) {
if count == 0 {
fix_start = Some(if remove_parentheses {
@ -403,7 +406,9 @@ pub fn remove_argument(
{
// Case 2: argument or keyword is _not_ the last node.
let mut seen_comma = false;
for (start, tok, end) in lexer::make_tokenizer_located(contents, stmt_at).flatten() {
for (start, tok, end) in
lexer::make_tokenizer_located(contents, Mode::Module, stmt_at).flatten()
{
if seen_comma {
if matches!(tok, Tok::NonLogicalNewline) {
// Also delete any non-logical newlines after the comma.
@ -426,7 +431,9 @@ pub fn remove_argument(
} else {
// Case 3: argument or keyword is the last node, so we have to find the last
// comma in the stmt.
for (start, tok, _) in lexer::make_tokenizer_located(contents, stmt_at).flatten() {
for (start, tok, _) in
lexer::make_tokenizer_located(contents, Mode::Module, stmt_at).flatten()
{
if start == expr_at {
fix_end = Some(expr_end);
break;

11
crates/ruff/src/checkers/logical_lines.rs
View file

@ -154,6 +154,7 @@ pub fn check_logical_lines(
mod tests {
use rustpython_parser::lexer;
use rustpython_parser::lexer::LexResult;
use rustpython_parser::mode::Mode;
use crate::checkers::logical_lines::iter_logical_lines;
use crate::source_code::Locator;
@ -164,7 +165,7 @@ mod tests {
x = 1
y = 2
z = x + 1"#;
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let locator = Locator::new(contents);
let actual: Vec<String> = iter_logical_lines(&lxr, &locator)
.into_iter()
@ -185,7 +186,7 @@ x = [
]
y = 2
z = x + 1"#;
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let locator = Locator::new(contents);
let actual: Vec<String> = iter_logical_lines(&lxr, &locator)
.into_iter()
@ -199,7 +200,7 @@ z = x + 1"#;
assert_eq!(actual, expected);
let contents = "x = 'abc'";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let locator = Locator::new(contents);
let actual: Vec<String> = iter_logical_lines(&lxr, &locator)
.into_iter()
@ -212,7 +213,7 @@ z = x + 1"#;
def f():
x = 1
f()"#;
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let locator = Locator::new(contents);
let actual: Vec<String> = iter_logical_lines(&lxr, &locator)
.into_iter()
@ -227,7 +228,7 @@ def f():
# Comment goes here.
x = 1
f()"#;
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let locator = Locator::new(contents);
let actual: Vec<String> = iter_logical_lines(&lxr, &locator)
.into_iter()

29
crates/ruff/src/directives.rs
View file

@ -152,6 +152,7 @@ mod tests {
use nohash_hasher::{IntMap, IntSet};
use rustpython_parser::lexer;
use rustpython_parser::lexer::LexResult;
use rustpython_parser::mode::Mode;
use crate::directives::{extract_isort_directives, extract_noqa_line_for};
@ -161,6 +162,7 @@ mod tests {
"x = 1
y = 2
z = x + 1",
Mode::Module,
)
.collect();
assert_eq!(extract_noqa_line_for(&lxr), IntMap::default());
@ -170,6 +172,7 @@ z = x + 1",
x = 1
y = 2
z = x + 1",
Mode::Module,
)
.collect();
assert_eq!(extract_noqa_line_for(&lxr), IntMap::default());
@ -179,6 +182,7 @@ z = x + 1",
y = 2
z = x + 1
",
Mode::Module,
)
.collect();
assert_eq!(extract_noqa_line_for(&lxr), IntMap::default());
@ -189,6 +193,7 @@ z = x + 1
y = 2
z = x + 1
",
Mode::Module,
)
.collect();
assert_eq!(extract_noqa_line_for(&lxr), IntMap::default());
@ -200,6 +205,7 @@ ghi
'''
y = 2
z = x + 1",
Mode::Module,
)
.collect();
assert_eq!(
@ -214,6 +220,7 @@ def
ghi
'''
z = 2",
Mode::Module,
)
.collect();
assert_eq!(
@ -227,6 +234,7 @@ y = '''abc
def
ghi
'''",
Mode::Module,
)
.collect();
assert_eq!(
@ -237,6 +245,7 @@ ghi
let lxr: Vec<LexResult> = lexer::make_tokenizer(
r#"x = \
1"#,
Mode::Module,
)
.collect();
assert_eq!(extract_noqa_line_for(&lxr), IntMap::from_iter([(1, 2)]));
@ -245,6 +254,7 @@ ghi
r#"from foo import \
bar as baz, \
qux as quux"#,
Mode::Module,
)
.collect();
assert_eq!(
@ -262,6 +272,7 @@ x = \
1
y = \
2"#,
Mode::Module,
)
.collect();
assert_eq!(
@ -275,7 +286,7 @@ y = \
let contents = "x = 1
y = 2
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(extract_isort_directives(&lxr).exclusions, IntSet::default());
let contents = "# isort: off
@ -283,7 +294,7 @@ x = 1
y = 2
# isort: on
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(
extract_isort_directives(&lxr).exclusions,
IntSet::from_iter([2, 3, 4])
@ -296,7 +307,7 @@ y = 2
# isort: on
z = x + 1
# isort: on";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(
extract_isort_directives(&lxr).exclusions,
IntSet::from_iter([2, 3, 4, 5])
@ -306,7 +317,7 @@ z = x + 1
x = 1
y = 2
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(
extract_isort_directives(&lxr).exclusions,
IntSet::from_iter([2, 3, 4])
@ -316,7 +327,7 @@ z = x + 1";
x = 1
y = 2
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(extract_isort_directives(&lxr).exclusions, IntSet::default());
let contents = "# isort: off
@ -325,7 +336,7 @@ x = 1
y = 2
# isort: skip_file
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(extract_isort_directives(&lxr).exclusions, IntSet::default());
}
@ -334,20 +345,20 @@ z = x + 1";
let contents = "x = 1
y = 2
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(extract_isort_directives(&lxr).splits, Vec::<usize>::new());
let contents = "x = 1
y = 2
# isort: split
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(extract_isort_directives(&lxr).splits, vec![3]);
let contents = "x = 1
y = 2 # isort: split
z = x + 1";
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
assert_eq!(extract_isort_directives(&lxr).splits, vec![2]);
}
}

5
crates/ruff/src/rules/flake8_annotations/fixes.rs
View file

@ -2,6 +2,7 @@ use anyhow::{bail, Result};
use rustpython_parser::ast::Stmt;
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use crate::ast::types::Range;
use crate::fix::Fix;
@ -16,7 +17,9 @@ pub fn add_return_none_annotation(locator: &Locator, stmt: &Stmt) -> Result<Fix>
let mut seen_lpar = false;
let mut seen_rpar = false;
let mut count: usize = 0;
for (start, tok, ..) in lexer::make_tokenizer_located(contents, range.location).flatten() {
for (start, tok, ..) in
lexer::make_tokenizer_located(contents, Mode::Module, range.location).flatten()
{
if seen_lpar && seen_rpar {
if matches!(tok, Tok::Colon) {
return Ok(Fix::insertion(" -> None".to_string(), start));

3
crates/ruff/src/rules/isort/comments.rs
View file

@ -3,6 +3,7 @@ use std::borrow::Cow;
use rustpython_parser::ast::Location;
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use crate::ast::types::Range;
use crate::source_code::Locator;
@ -17,7 +18,7 @@ pub struct Comment<'a> {
/// Collect all comments in an import block.
pub fn collect_comments<'a>(range: &Range, locator: &'a Locator) -> Vec<Comment<'a>> {
let contents = locator.slice(range);
lexer::make_tokenizer_located(contents, range.location)
lexer::make_tokenizer_located(contents, Mode::Module, range.location)
.flatten()
.filter_map(|(start, tok, end)| {
if let Tok::Comment(value) = tok {

13
crates/ruff/src/rules/isort/helpers.rs
View file

@ -1,19 +1,23 @@
use rustpython_parser::ast::{Location, Stmt};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use super::types::TrailingComma;
use crate::ast::helpers::is_docstring_stmt;
use crate::ast::types::Range;
use crate::source_code::Locator;
use super::types::TrailingComma;
/// Return `true` if a `StmtKind::ImportFrom` statement ends with a magic
/// trailing comma.
pub fn trailing_comma(stmt: &Stmt, locator: &Locator) -> TrailingComma {
let contents = locator.slice(&Range::from_located(stmt));
let mut count: usize = 0;
let mut trailing_comma = TrailingComma::Absent;
for (_, tok, _) in lexer::make_tokenizer(contents).flatten() {
for (_, tok, _) in
lexer::make_tokenizer_located(contents, Mode::Module, stmt.location).flatten()
{
if matches!(tok, Tok::Lpar) {
count += 1;
}
@ -110,7 +114,7 @@ pub fn find_splice_location(body: &[Stmt], locator: &Locator) -> Location {
// Find the first token that isn't a comment or whitespace.
let contents = locator.skip(splice);
for (.., tok, end) in lexer::make_tokenizer_located(contents, splice).flatten() {
for (.., tok, end) in lexer::make_tokenizer_located(contents, Mode::Module, splice).flatten() {
if matches!(tok, Tok::Comment(..) | Tok::Newline) {
splice = end;
} else {
@ -127,9 +131,10 @@ mod tests {
use rustpython_parser::ast::Location;
use rustpython_parser::parser;
use super::find_splice_location;
use crate::source_code::Locator;
use super::find_splice_location;
fn splice_contents(contents: &str) -> Result<Location> {
let program = parser::parse_program(contents, "<filename>")?;
let locator = Locator::new(contents);

6
crates/ruff/src/rules/pyflakes/fixes.rs
View file

@ -1,9 +1,11 @@
use anyhow::{bail, Result};
use libcst_native::{Call, Codegen, CodegenState, Dict, DictElement, Expression};
use ruff_python::string::strip_quotes_and_prefixes;
use rustpython_parser::ast::{Excepthandler, Expr};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use ruff_python::string::strip_quotes_and_prefixes;
use crate::ast::types::Range;
use crate::cst::matchers::{match_expr, match_module};
@ -122,7 +124,7 @@ pub fn remove_exception_handler_assignment(
// End of the token just before the `as` to the semicolon.
let mut prev = None;
for (start, tok, end) in
lexer::make_tokenizer_located(contents, excepthandler.location).flatten()
lexer::make_tokenizer_located(contents, Mode::Module, excepthandler.location).flatten()
{
if matches!(tok, Tok::As) {
fix_start = prev;

8
crates/ruff/src/rules/pyflakes/rules/unused_variable.rs
View file

@ -3,6 +3,7 @@ use log::error;
use rustpython_parser::ast::{ExprKind, Located, Stmt, StmtKind};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use ruff_macros::{define_violation, derive_message_formats};
@ -74,7 +75,8 @@ where
let mut sqb_count = 0;
let mut brace_count = 0;
for ((_, tok, _), (start, _, end)) in lexer::make_tokenizer_located(contents, located.location)
for ((_, tok, _), (start, _, end)) in
lexer::make_tokenizer_located(contents, Mode::Module, located.location)
.flatten()
.tuple_windows()
{
@ -136,7 +138,9 @@ where
let mut sqb_count = 0;
let mut brace_count = 0;
for (start, tok, end) in lexer::make_tokenizer_located(contents, located.location).flatten() {
for (start, tok, end) in
lexer::make_tokenizer_located(contents, Mode::Module, located.location).flatten()
{
match tok {
Tok::Lpar => {
par_count += 1;

5
crates/ruff/src/rules/pylint/rules/bad_string_format_type.rs
View file

@ -5,6 +5,7 @@ use rustpython_common::cformat::{CFormatPart, CFormatSpec, CFormatStrOrBytes, CF
use rustpython_parser::ast::{Constant, Expr, ExprKind, Location, Operator};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use ruff_macros::{define_violation, derive_message_formats};
@ -247,7 +248,9 @@ pub fn bad_string_format_type(checker: &mut Checker, expr: &Expr, right: &Expr)
// Grab each string segment (in case there's an implicit concatenation).
let content = checker.locator.slice(&Range::from_located(expr));
let mut strings: Vec<(Location, Location)> = vec![];
for (start, tok, end) in lexer::make_tokenizer_located(content, expr.location).flatten() {
for (start, tok, end) in
lexer::make_tokenizer_located(content, Mode::Module, expr.location).flatten()
{
if matches!(tok, Tok::String { .. }) {
strings.push((start, end));
} else if matches!(tok, Tok::Percent) {

3
crates/ruff/src/rules/pyupgrade/fixes.rs
View file

@ -6,6 +6,7 @@ use libcst_native::{
use rustpython_parser::ast::{Expr, Keyword, Location};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use crate::ast::types::Range;
use crate::autofix::helpers::remove_argument;
@ -110,7 +111,7 @@ pub fn remove_import_members(contents: &str, members: &[&str]) -> String {
// Find all Tok::Name tokens that are not preceded by Tok::As, and all
// Tok::Comma tokens.
let mut prev_tok = None;
for (start, tok, end) in lexer::make_tokenizer(contents)
for (start, tok, end) in lexer::make_tokenizer(contents, Mode::Module)
.flatten()
.skip_while(|(_, tok, _)| !matches!(tok, Tok::Import))
{

6
crates/ruff/src/rules/pyupgrade/rules/f_strings.rs
View file

@ -1,4 +1,3 @@
use ruff_macros::{define_violation, derive_message_formats};
use rustc_hash::FxHashMap;
use rustpython_common::format::{
FieldName, FieldNamePart, FieldType, FormatPart, FormatString, FromTemplate,
@ -6,6 +5,9 @@ use rustpython_common::format::{
use rustpython_parser::ast::{Constant, Expr, ExprKind, KeywordData};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use ruff_macros::{define_violation, derive_message_formats};
use crate::ast::types::Range;
use crate::checkers::ast::Checker;
@ -129,7 +131,7 @@ fn try_convert_to_f_string(checker: &Checker, expr: &Expr) -> Option<String> {
let contents = checker.locator.slice(&Range::from_located(value));
// Tokenize: we need to avoid trying to fix implicit string concatenations.
if lexer::make_tokenizer(contents)
if lexer::make_tokenizer(contents, Mode::Module)
.flatten()
.filter(|(_, tok, _)| matches!(tok, Tok::String { .. }))
.count()

3
crates/ruff/src/rules/pyupgrade/rules/native_literals.rs
View file

@ -4,6 +4,7 @@ use ruff_macros::{define_violation, derive_message_formats};
use rustpython_parser::ast::{Constant, Expr, ExprKind, Keyword};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use serde::{Deserialize, Serialize};
use crate::ast::types::Range;
@ -118,7 +119,7 @@ pub fn native_literals(
// safely remove the outer call in this situation. We're following pyupgrade
// here and skip.
let arg_code = checker.locator.slice(&Range::from_located(arg));
if lexer::make_tokenizer(arg_code)
if lexer::make_tokenizer_located(arg_code, Mode::Module, arg.location)
.flatten()
.filter(|(_, tok, _)| matches!(tok, Tok::String { .. }))
.count()

6
crates/ruff/src/rules/pyupgrade/rules/outdated_version_block.rs
View file

@ -2,10 +2,12 @@ use std::cmp::Ordering;
use log::error;
use num_bigint::{BigInt, Sign};
use ruff_macros::{define_violation, derive_message_formats};
use rustpython_parser::ast::{Cmpop, Constant, Expr, ExprKind, Located, Location, Stmt};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use ruff_macros::{define_violation, derive_message_formats};
use crate::ast::types::{Range, RefEquality};
use crate::ast::whitespace::indentation;
@ -67,7 +69,7 @@ fn metadata<T>(locator: &Locator, located: &Located<T>) -> Option<BlockMetadata>
let mut else_ = None;
for (start, tok, _) in
lexer::make_tokenizer_located(text, Location::new(located.location.row(), 0))
lexer::make_tokenizer_located(text, Mode::Module, Location::new(located.location.row(), 0))
.flatten()
.filter(|(_, tok, _)| {
!matches!(

9
crates/ruff/src/rules/pyupgrade/rules/printf_string_formatting.rs
View file

@ -1,14 +1,16 @@
use std::str::FromStr;
use ruff_macros::{define_violation, derive_message_formats};
use ruff_python::identifiers::is_identifier;
use ruff_python::keyword::KWLIST;
use rustpython_common::cformat::{
CConversionFlags, CFormatPart, CFormatPrecision, CFormatQuantity, CFormatString,
};
use rustpython_parser::ast::{Constant, Expr, ExprKind, Location};
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use ruff_macros::{define_violation, derive_message_formats};
use ruff_python::identifiers::is_identifier;
use ruff_python::keyword::KWLIST;
use crate::ast::types::Range;
use crate::ast::whitespace::indentation;
@ -321,6 +323,7 @@ pub(crate) fn printf_string_formatting(
let mut extension = None;
for (start, tok, end) in lexer::make_tokenizer_located(
checker.locator.slice(&Range::from_located(expr)),
Mode::Module,
expr.location,
)
.flatten()

5
crates/ruff/src/rules/pyupgrade/rules/redundant_open_modes.rs
View file

@ -5,6 +5,7 @@ use log::error;
use ruff_macros::{define_violation, derive_message_formats};
use rustpython_parser::ast::{Constant, Expr, ExprKind, Keyword, Location};
use rustpython_parser::lexer;
use rustpython_parser::mode::Mode;
use rustpython_parser::token::Tok;
use crate::ast::helpers::find_keyword;
@ -142,7 +143,9 @@ fn create_remove_param_fix(locator: &Locator, expr: &Expr, mode_param: &Expr) ->
let mut fix_end: Option<Location> = None;
let mut is_first_arg: bool = false;
let mut delete_first_arg: bool = false;
for (start, tok, end) in lexer::make_tokenizer_located(content, expr.location).flatten() {
for (start, tok, end) in
lexer::make_tokenizer_located(content, Mode::Module, expr.location).flatten()
{
if start == mode_param.location {
if is_first_arg {
delete_first_arg = true;

2
crates/ruff/src/rustpython_helpers.rs
View file

@ -7,7 +7,7 @@ use rustpython_parser::{lexer, parser};
/// Collect tokens up to and including the first error.
pub fn tokenize(contents: &str) -> Vec<LexResult> {
let mut tokens: Vec<LexResult> = vec![];
for tok in lexer::make_tokenizer(contents) {
for tok in lexer::make_tokenizer(contents, Mode::Module) {
let is_err = tok.is_err();
tokens.push(tok);
if is_err {

9
crates/ruff/src/source_code/indexer.rs
View file

@ -51,13 +51,14 @@ impl From<&[LexResult]> for Indexer {
mod tests {
use rustpython_parser::lexer;
use rustpython_parser::lexer::LexResult;
use rustpython_parser::mode::Mode;
use crate::source_code::Indexer;
#[test]
fn continuation() {
let contents = r#"x = 1"#;
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let indexer: Indexer = lxr.as_slice().into();
assert_eq!(indexer.continuation_lines(), Vec::<usize>::new().as_slice());
@ -69,7 +70,7 @@ x = 1
y = 2
"#
.trim();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let indexer: Indexer = lxr.as_slice().into();
assert_eq!(indexer.continuation_lines(), Vec::<usize>::new().as_slice());
@ -89,7 +90,7 @@ if True:
)
"#
.trim();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let indexer: Indexer = lxr.as_slice().into();
assert_eq!(indexer.continuation_lines(), [1, 5, 6, 11]);
@ -109,7 +110,7 @@ x = 1; \
import os
"#
.trim();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents).collect();
let lxr: Vec<LexResult> = lexer::make_tokenizer(contents, Mode::Module).collect();
let indexer: Indexer = lxr.as_slice().into();
assert_eq!(indexer.continuation_lines(), [9, 12]);
}

5
crates/ruff/src/source_code/stylist.rs
View file

@ -7,6 +7,7 @@ use once_cell::unsync::OnceCell;
use rustpython_parser::ast::Location;
use rustpython_parser::lexer;
use rustpython_parser::lexer::Tok;
use rustpython_parser::mode::Mode;
use crate::ast::types::Range;
use crate::rules::pydocstyle::helpers::leading_quote;
@ -165,7 +166,7 @@ impl Deref for LineEnding {
/// Detect the indentation style of the given tokens.
fn detect_indentation(contents: &str, locator: &Locator) -> Option<Indentation> {
for (_start, tok, end) in lexer::make_tokenizer(contents).flatten() {
for (_start, tok, end) in lexer::make_tokenizer(contents, Mode::Module).flatten() {
if let Tok::Indent { .. } = tok {
let start = Location::new(end.row(), 0);
let whitespace = locator.slice(&Range::new(start, end));
@ -177,7 +178,7 @@ fn detect_indentation(contents: &str, locator: &Locator) -> Option<Indentation>
/// Detect the quotation style of the given tokens.
fn detect_quote(contents: &str, locator: &Locator) -> Option<Quote> {
for (start, tok, end) in lexer::make_tokenizer(contents).flatten() {
for (start, tok, end) in lexer::make_tokenizer(contents, Mode::Module).flatten() {
if let Tok::String { .. } = tok {
let content = locator.slice(&Range::new(start, end));
if let Some(pattern) = leading_quote(content) {

3
crates/ruff_dev/src/print_tokens.rs
View file

@ -6,6 +6,7 @@ use std::path::PathBuf;
use anyhow::Result;
use rustpython_parser::lexer;
use rustpython_parser::mode::Mode;
#[derive(clap::Args)]
pub struct Args {
@ -16,7 +17,7 @@ pub struct Args {
pub fn main(args: &Args) -> Result<()> {
let contents = fs::read_to_string(&args.file)?;
for (_, tok, _) in lexer::make_tokenizer(&contents).flatten() {
for (_, tok, _) in lexer::make_tokenizer(&contents, Mode::Module).flatten() {
println!("{tok:#?}");
}
Ok(())

2
crates/ruff_python_formatter/src/core/rustpython_helpers.rs
View file

@ -7,7 +7,7 @@ use rustpython_parser::{lexer, parser};
/// Collect tokens up to and including the first error.
pub fn tokenize(contents: &str) -> Vec<LexResult> {
let mut tokens: Vec<LexResult> = vec![];
for tok in lexer::make_tokenizer(contents) {
for tok in lexer::make_tokenizer(contents, Mode::Module) {
let is_err = tok.is_err();
tokens.push(tok);
if is_err {