diff --git a/Cargo.lock b/Cargo.lock index 8e13417069..11293fa2b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1985,6 +1985,7 @@ dependencies = [ "result-like", "ruff_macros", "ruff_python", + "ruff_rustpython", "rustc-hash", "rustpython-common", "rustpython-parser", @@ -2109,6 +2110,7 @@ dependencies = [ "insta", "once_cell", "ruff_formatter", + "ruff_rustpython", "ruff_testing_macros", "ruff_text_size", "rustc-hash", @@ -2118,6 +2120,16 @@ dependencies = [ "test-case", ] +[[package]] +name = "ruff_rustpython" +version = "0.0.0" +dependencies = [ + "anyhow", + "once_cell", + "rustpython-common", + "rustpython-parser", +] + [[package]] name = "ruff_testing_macros" version = "0.0.0" diff --git a/crates/ruff/Cargo.toml b/crates/ruff/Cargo.toml index 5cf8cc2844..e100007472 100644 --- a/crates/ruff/Cargo.toml +++ b/crates/ruff/Cargo.toml @@ -16,6 +16,10 @@ crate-type = ["cdylib", "rlib"] doctest = false [dependencies] +ruff_macros = { path = "../ruff_macros" } +ruff_python = { path = "../ruff_python" } +ruff_rustpython = { path = "../ruff_rustpython" } + anyhow = { workspace = true } bisection = { version = "0.1.0" } bitflags = { version = "1.3.2" } @@ -40,8 +44,6 @@ once_cell = { workspace = true } path-absolutize = { version = "3.0.14", features = ["once_cell_cache", "use_unix_paths_on_wasm"] } regex = { workspace = true } result-like = "0.4.6" -ruff_macros = { path = "../ruff_macros" } -ruff_python = { path = "../ruff_python" } rustc-hash = { workspace = true } rustpython-common = { workspace = true } rustpython-parser = { workspace = true } diff --git a/crates/ruff/src/lib.rs b/crates/ruff/src/lib.rs index a1f93144a1..df85cd0838 100644 --- a/crates/ruff/src/lib.rs +++ b/crates/ruff/src/lib.rs @@ -33,10 +33,8 @@ pub mod resolver; mod rule_redirects; mod rule_selector; mod rules; -mod rustpython_helpers; pub mod settings; pub mod source_code; -mod vendor; mod violation; mod visibility; diff --git a/crates/ruff/src/lib_wasm.rs b/crates/ruff/src/lib_wasm.rs index 203c262171..fcbcb79a52 100644 --- a/crates/ruff/src/lib_wasm.rs +++ b/crates/ruff/src/lib_wasm.rs @@ -14,7 +14,6 @@ use crate::rules::{ flake8_quotes, flake8_self, flake8_tidy_imports, flake8_type_checking, flake8_unused_arguments, isort, mccabe, pep8_naming, pycodestyle, pydocstyle, pylint, pyupgrade, }; -use crate::rustpython_helpers::tokenize; use crate::settings::configuration::Configuration; use crate::settings::options::Options; use crate::settings::{defaults, flags, Settings}; @@ -175,7 +174,7 @@ pub fn check(contents: &str, options: JsValue) -> Result { Settings::from_configuration(configuration, Path::new(".")).map_err(|e| e.to_string())?; // Tokenize once. - let tokens: Vec = tokenize(contents); + let tokens: Vec = ruff_rustpython::tokenize(contents); // Map row and column locations to byte slices (lazily). let locator = Locator::new(contents); diff --git a/crates/ruff/src/linter.rs b/crates/ruff/src/linter.rs index 2737d64f58..f16bf31d7d 100644 --- a/crates/ruff/src/linter.rs +++ b/crates/ruff/src/linter.rs @@ -24,7 +24,7 @@ use crate::registry::{Diagnostic, LintSource, Rule}; use crate::rules::pycodestyle; use crate::settings::{flags, Settings}; use crate::source_code::{Indexer, Locator, Stylist}; -use crate::{directives, fs, rustpython_helpers}; +use crate::{directives, fs}; const CARGO_PKG_NAME: &str = env!("CARGO_PKG_NAME"); const CARGO_PKG_REPOSITORY: &str = env!("CARGO_PKG_REPOSITORY"); @@ -115,7 +115,7 @@ pub fn check_path( .iter_enabled() .any(|rule_code| matches!(rule_code.lint_source(), LintSource::Imports)); if use_ast || use_imports || use_doc_lines { - match rustpython_helpers::parse_program_tokens(tokens, &path.to_string_lossy()) { + match ruff_rustpython::parse_program_tokens(tokens, &path.to_string_lossy()) { Ok(python_ast) => { if use_ast { diagnostics.extend(check_ast( @@ -226,7 +226,7 @@ pub fn add_noqa_to_path(path: &Path, package: Option<&Path>, settings: &Settings let contents = std::fs::read_to_string(path)?; // Tokenize once. - let tokens: Vec = rustpython_helpers::tokenize(&contents); + let tokens: Vec = ruff_rustpython::tokenize(&contents); // Map row and column locations to byte slices (lazily). let locator = Locator::new(&contents); @@ -290,7 +290,7 @@ pub fn lint_only( autofix: flags::Autofix, ) -> LinterResult> { // Tokenize once. - let tokens: Vec = rustpython_helpers::tokenize(contents); + let tokens: Vec = ruff_rustpython::tokenize(contents); // Map row and column locations to byte slices (lazily). let locator = Locator::new(contents); @@ -359,7 +359,7 @@ pub fn lint_fix<'a>( // Continuously autofix until the source code stabilizes. loop { // Tokenize once. - let tokens: Vec = rustpython_helpers::tokenize(&transformed); + let tokens: Vec = ruff_rustpython::tokenize(&transformed); // Map row and column locations to byte slices (lazily). let locator = Locator::new(&transformed); diff --git a/crates/ruff/src/rules/pandas_vet/mod.rs b/crates/ruff/src/rules/pandas_vet/mod.rs index 7de30c7043..8e9f8a6171 100644 --- a/crates/ruff/src/rules/pandas_vet/mod.rs +++ b/crates/ruff/src/rules/pandas_vet/mod.rs @@ -18,12 +18,12 @@ mod tests { use crate::settings::flags; use crate::source_code::{Indexer, Locator, Stylist}; use crate::test::test_path; - use crate::{directives, rustpython_helpers, settings}; + use crate::{directives, settings}; fn rule_code(contents: &str, expected: &[Rule]) { let contents = dedent(contents); let settings = settings::Settings::for_rules(&Linter::PandasVet); - let tokens: Vec = rustpython_helpers::tokenize(&contents); + let tokens: Vec = ruff_rustpython::tokenize(&contents); let locator = Locator::new(&contents); let stylist = Stylist::from_contents(&contents, &locator); let indexer: Indexer = tokens.as_slice().into(); diff --git a/crates/ruff/src/rules/pyflakes/mod.rs b/crates/ruff/src/rules/pyflakes/mod.rs index 6db17a0888..a63cdf7f61 100644 --- a/crates/ruff/src/rules/pyflakes/mod.rs +++ b/crates/ruff/src/rules/pyflakes/mod.rs @@ -20,7 +20,7 @@ mod tests { use crate::settings::flags; use crate::source_code::{Indexer, Locator, Stylist}; use crate::test::test_path; - use crate::{directives, rustpython_helpers, settings}; + use crate::{directives, settings}; #[test_case(Rule::UnusedImport, Path::new("F401_0.py"); "F401_0")] #[test_case(Rule::UnusedImport, Path::new("F401_1.py"); "F401_1")] @@ -243,7 +243,7 @@ mod tests { fn flakes(contents: &str, expected: &[Rule]) { let contents = dedent(contents); let settings = settings::Settings::for_rules(&Linter::Pyflakes); - let tokens: Vec = rustpython_helpers::tokenize(&contents); + let tokens: Vec = ruff_rustpython::tokenize(&contents); let locator = Locator::new(&contents); let stylist = Stylist::from_contents(&contents, &locator); let indexer: Indexer = tokens.as_slice().into(); diff --git a/crates/ruff/src/source_code/generator.rs b/crates/ruff/src/source_code/generator.rs index 33071934bf..408bde05ba 100644 --- a/crates/ruff/src/source_code/generator.rs +++ b/crates/ruff/src/source_code/generator.rs @@ -8,8 +8,9 @@ use rustpython_parser::ast::{ Suite, Withitem, }; +use ruff_rustpython::vendor::{bytes, str}; + use crate::source_code::stylist::{Indentation, LineEnding, Quote, Stylist}; -use crate::vendor::{bytes, str}; mod precedence { pub const ASSIGN: u8 = 3; diff --git a/crates/ruff/src/source_code/stylist.rs b/crates/ruff/src/source_code/stylist.rs index 73a022d848..8b4c726566 100644 --- a/crates/ruff/src/source_code/stylist.rs +++ b/crates/ruff/src/source_code/stylist.rs @@ -4,13 +4,13 @@ use std::fmt; use std::ops::Deref; use once_cell::unsync::OnceCell; +use ruff_rustpython::vendor; use rustpython_parser::ast::Location; use rustpython_parser::{lexer, Mode, Tok}; use crate::ast::types::Range; use crate::rules::pydocstyle::helpers::leading_quote; use crate::source_code::Locator; -use crate::vendor; pub struct Stylist<'a> { contents: &'a str, diff --git a/crates/ruff/src/test.rs b/crates/ruff/src/test.rs index c1bc2fe445..fb58227714 100644 --- a/crates/ruff/src/test.rs +++ b/crates/ruff/src/test.rs @@ -7,12 +7,12 @@ use anyhow::Result; use rustpython_parser::lexer::LexResult; use crate::autofix::fix_file; +use crate::directives; use crate::linter::{check_path, LinterResult}; use crate::packaging::detect_package_root; use crate::registry::Diagnostic; use crate::settings::{flags, Settings}; use crate::source_code::{Indexer, Locator, Stylist}; -use crate::{directives, rustpython_helpers}; pub fn test_resource_path(path: impl AsRef) -> std::path::PathBuf { Path::new("./resources/test/").join(path) @@ -23,7 +23,7 @@ pub fn test_resource_path(path: impl AsRef) -> std::path::PathBuf { pub fn test_path(path: &Path, settings: &Settings) -> Result> { let path = test_resource_path("fixtures").join(path); let contents = std::fs::read_to_string(&path)?; - let tokens: Vec = rustpython_helpers::tokenize(&contents); + let tokens: Vec = ruff_rustpython::tokenize(&contents); let locator = Locator::new(&contents); let stylist = Stylist::from_contents(&contents, &locator); let indexer: Indexer = tokens.as_slice().into(); @@ -58,7 +58,7 @@ pub fn test_path(path: &Path, settings: &Settings) -> Result> { let mut iterations = 0; loop { - let tokens: Vec = rustpython_helpers::tokenize(&contents); + let tokens: Vec = ruff_rustpython::tokenize(&contents); let locator = Locator::new(&contents); let stylist = Stylist::from_contents(&contents, &locator); let indexer: Indexer = tokens.as_slice().into(); diff --git a/crates/ruff_python_formatter/Cargo.toml b/crates/ruff_python_formatter/Cargo.toml index a7e76f31f9..5d08682017 100644 --- a/crates/ruff_python_formatter/Cargo.toml +++ b/crates/ruff_python_formatter/Cargo.toml @@ -6,17 +6,20 @@ edition = { workspace = true } rust-version = { workspace = true } [dependencies] +ruff_formatter = { path = "../ruff_formatter" } +ruff_rustpython = { path = "../ruff_rustpython" } +ruff_text_size = { path = "../ruff_text_size" } + anyhow = { workspace = true } clap = { workspace = true } once_cell = { workspace = true } -ruff_formatter = { path = "../ruff_formatter" } -ruff_text_size = { path = "../ruff_text_size" } rustc-hash = { workspace = true } rustpython-common = { workspace = true } rustpython-parser = { workspace = true } [dev-dependencies] +ruff_testing_macros = { path = "../ruff_testing_macros" } + insta = { version = "1.19.0", features = [] } test-case = { version = "2.2.2" } -ruff_testing_macros = { path = "../ruff_testing_macros" } -similar = "2.2.1" +similar = { version = "2.2.1" } diff --git a/crates/ruff_python_formatter/src/core/mod.rs b/crates/ruff_python_formatter/src/core/mod.rs index ac97bd01f3..3812c9abd1 100644 --- a/crates/ruff_python_formatter/src/core/mod.rs +++ b/crates/ruff_python_formatter/src/core/mod.rs @@ -1,4 +1,3 @@ pub mod locator; -pub mod rustpython_helpers; pub mod types; pub mod visitor; diff --git a/crates/ruff_python_formatter/src/core/rustpython_helpers.rs b/crates/ruff_python_formatter/src/core/rustpython_helpers.rs deleted file mode 100644 index 4f5638a423..0000000000 --- a/crates/ruff_python_formatter/src/core/rustpython_helpers.rs +++ /dev/null @@ -1,28 +0,0 @@ -use rustpython_parser as parser; -use rustpython_parser::ast::{Mod, Suite}; -use rustpython_parser::lexer::LexResult; -use rustpython_parser::{lexer, Mode, ParseError}; - -/// Collect tokens up to and including the first error. -pub fn tokenize(contents: &str) -> Vec { - let mut tokens: Vec = vec![]; - for tok in lexer::lex(contents, Mode::Module) { - let is_err = tok.is_err(); - tokens.push(tok); - if is_err { - break; - } - } - tokens -} - -/// Parse a full Python program from its tokens. -pub(crate) fn parse_program_tokens( - lxr: Vec, - source_path: &str, -) -> anyhow::Result { - parser::parse_tokens(lxr, Mode::Module, source_path).map(|top| match top { - Mod::Module { body, .. } => body, - _ => unreachable!(), - }) -} diff --git a/crates/ruff_python_formatter/src/lib.rs b/crates/ruff_python_formatter/src/lib.rs index a960c60b32..9a263fc41a 100644 --- a/crates/ruff_python_formatter/src/lib.rs +++ b/crates/ruff_python_formatter/src/lib.rs @@ -5,7 +5,6 @@ use rustpython_parser::lexer::LexResult; use crate::attachment::attach; use crate::context::ASTFormatContext; use crate::core::locator::Locator; -use crate::core::rustpython_helpers; use crate::cst::Stmt; use crate::newlines::normalize_newlines; use crate::parentheses::normalize_parentheses; @@ -24,13 +23,13 @@ pub mod trivia; pub fn fmt(contents: &str) -> Result> { // Tokenize once. - let tokens: Vec = rustpython_helpers::tokenize(contents); + let tokens: Vec = ruff_rustpython::tokenize(contents); // Extract trivia. let trivia = trivia::extract_trivia_tokens(&tokens); // Parse the AST. - let python_ast = rustpython_helpers::parse_program_tokens(tokens, "")?; + let python_ast = ruff_rustpython::parse_program_tokens(tokens, "")?; // Convert to a CST. let mut python_cst: Vec = python_ast.into_iter().map(Into::into).collect(); diff --git a/crates/ruff_rustpython/Cargo.toml b/crates/ruff_rustpython/Cargo.toml new file mode 100644 index 0000000000..3c4b893d07 --- /dev/null +++ b/crates/ruff_rustpython/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "ruff_rustpython" +version = "0.0.0" +publish = false +edition = { workspace = true } +rust-version = { workspace = true } + +[lib] + +[dependencies] +anyhow = { workspace = true } +once_cell = { workspace = true } +rustpython-common = { workspace = true } +rustpython-parser = { workspace = true } diff --git a/crates/ruff/src/rustpython_helpers.rs b/crates/ruff_rustpython/src/lib.rs similarity index 94% rename from crates/ruff/src/rustpython_helpers.rs rename to crates/ruff_rustpython/src/lib.rs index 4f5638a423..a40a9db739 100644 --- a/crates/ruff/src/rustpython_helpers.rs +++ b/crates/ruff_rustpython/src/lib.rs @@ -3,6 +3,8 @@ use rustpython_parser::ast::{Mod, Suite}; use rustpython_parser::lexer::LexResult; use rustpython_parser::{lexer, Mode, ParseError}; +pub mod vendor; + /// Collect tokens up to and including the first error. pub fn tokenize(contents: &str) -> Vec { let mut tokens: Vec = vec![]; @@ -17,7 +19,7 @@ pub fn tokenize(contents: &str) -> Vec { } /// Parse a full Python program from its tokens. -pub(crate) fn parse_program_tokens( +pub fn parse_program_tokens( lxr: Vec, source_path: &str, ) -> anyhow::Result { diff --git a/crates/ruff/src/vendor/bytes.rs b/crates/ruff_rustpython/src/vendor/bytes.rs similarity index 100% rename from crates/ruff/src/vendor/bytes.rs rename to crates/ruff_rustpython/src/vendor/bytes.rs diff --git a/crates/ruff/src/vendor/mod.rs b/crates/ruff_rustpython/src/vendor/mod.rs similarity index 100% rename from crates/ruff/src/vendor/mod.rs rename to crates/ruff_rustpython/src/vendor/mod.rs diff --git a/crates/ruff/src/vendor/str.rs b/crates/ruff_rustpython/src/vendor/str.rs similarity index 100% rename from crates/ruff/src/vendor/str.rs rename to crates/ruff_rustpython/src/vendor/str.rs