Create a separate dev crate for development scripts (#607)

Cargo.lock

@@ -2230,7 +2230,6 @@ dependencies = [
  "chrono",
  "clap 4.0.15",
  "clearscreen",
- "codegen",
  "colored",
  "common-path",
  "criterion",
@@ -2265,6 +2264,22 @@ dependencies = [
  "walkdir",
 ]
 
+[[package]]
+name = "ruff_dev"
+version = "0.0.101"
+dependencies = [
+ "anyhow",
+ "clap 4.0.15",
+ "codegen",
+ "itertools",
+ "ruff",
+ "rustpython-ast",
+ "rustpython-common",
+ "rustpython-parser",
+ "strum",
+ "strum_macros",
+]
+
 [[package]]
 name = "rustls"
 version = "0.20.6"

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 members = [
   "flake8_to_ruff",
+  "ruff_dev",
 ]
 
 [package]
@@ -56,7 +57,6 @@ getrandom = { version = "0.2.7", features = ["js"] }
 
 [dev-dependencies]
 assert_cmd = { version = "2.0.4" }
-codegen = { version = "0.2.0" }
 criterion = { version = "0.4.0" }
 insta = { version = "1.19.1", features = ["yaml"] }
 test-case = { version = "2.2.2" }

ruff_dev/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "ruff_dev"
+version = "0.0.101"
+edition = "2021"
+
+[dependencies]
+anyhow = { version = "1.0.60" }
+clap = { version = "4.0.1", features = ["derive"] }
+codegen = { version = "0.2.0" }
+itertools = { version = "0.10.5" }
+ruff = { path = ".." }
+rustpython-ast = { features = ["unparse"], git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" }
+rustpython-common = { git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" }
+rustpython-parser = { features = ["lalrpop"], git = "https://github.com/RustPython/RustPython.git", rev = "77b821a1941019fe34f73ce17cea013ae1b98fd0" }
+strum = { version = "0.24.1", features = ["strum_macros"] }
+strum_macros = { version = "0.24.3" }

ruff_dev/src/generate_check_code_prefix.rs

@@ -1,3 +1,5 @@
+//! Generate the `CheckCodePrefix` enum.
+
 use std::collections::{BTreeMap, BTreeSet};
 use std::fs::OpenOptions;
 use std::io::Write;
@@ -13,17 +15,14 @@ const FILE: &str = "src/checks_gen.rs";
 
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
-/// Generate the `CheckCodePrefix` enum.
+pub struct Cli {
-struct Cli {
     /// Write the generated source code to stdout (rather than to
     /// `src/checks_gen.rs`).
     #[arg(long)]
     dry_run: bool,
 }
 
-fn main() -> Result<()> {
+pub fn main(cli: &Cli) -> Result<()> {
-    let cli = Cli::parse();
-
     // Build up a map from prefix to matching CheckCodes.
     let mut prefix_to_codes: BTreeMap<String, BTreeSet<CheckCode>> = Default::default();
     for check_code in CheckCode::iter() {
@@ -34,9 +33,7 @@ fn main() -> Result<()> {
             .count();
         for i in prefix_len..=as_ref.len() {
             let prefix = as_ref[..i].to_string();
-            let entry = prefix_to_codes
+            let entry = prefix_to_codes.entry(prefix).or_default();
-                .entry(prefix)
-                .or_insert_with(|| Default::default());
             entry.insert(check_code.clone());
         }
     }
@@ -56,7 +53,7 @@ fn main() -> Result<()> {
         .derive("Clone")
         .derive("Serialize")
         .derive("Deserialize");
-    for (prefix, _) in &prefix_to_codes {
+    for prefix in prefix_to_codes.keys() {
         gen = gen.push_variant(Variant::new(prefix.to_string()));
     }
 
@@ -100,7 +97,7 @@ fn main() -> Result<()> {
         .ret(Type::new("PrefixSpecificity"))
         .vis("pub")
         .line("match self {");
-    for (prefix, _) in &prefix_to_codes {
+    for prefix in prefix_to_codes.keys() {
         let num_numeric = prefix.chars().filter(|char| char.is_numeric()).count();
         let specificity = match num_numeric {
             3 => "Explicit",
@@ -129,7 +126,7 @@ fn main() -> Result<()> {
     output.push_str("use crate::checks::CheckCode;");
     output.push('\n');
     output.push('\n');
-    output.push_str(&format!("{}", scope.to_string()));
+    output.push_str(&scope.to_string());
 
     // Write the output to `src/checks_gen.rs` (or stdout).
     if cli.dry_run {

ruff_dev/src/generate_rules_table.rs

@@ -1,28 +1,26 @@
+//! Generate a Markdown-compatible table of supported lint rules.
+
 use std::fs;
 use std::fs::OpenOptions;
 use std::io::Write;
 
 use anyhow::Result;
-use clap::Parser;
+use clap::Args;
 use ruff::checks::{CheckCategory, CheckCode};
 use strum::IntoEnumIterator;
 
-const FILE: &str = "README.md";
+const FILE: &str = "../README.md";
 const BEGIN_PRAGMA: &str = "<!-- Begin auto-generated sections. -->";
 const END_PRAGMA: &str = "<!-- End auto-generated sections. -->";
 
-#[derive(Parser)]
+#[derive(Args)]
-#[command(author, version, about, long_about = None)]
+pub struct Cli {
-/// Generate a Markdown-compatible table of supported lint rules.
-struct Cli {
     /// Write the generated table to stdout (rather than to `README.md`).
     #[arg(long)]
     dry_run: bool,
 }
 
-fn main() -> Result<()> {
+pub fn main(cli: &Cli) -> Result<()> {
-    let cli = Cli::parse();
-
     // Generate the table string.
     let mut output = String::new();
     for check_category in CheckCategory::iter() {
@@ -43,7 +41,7 @@ fn main() -> Result<()> {
                     "| {} | {} | {} | {} |",
                     check_kind.code().as_ref(),
                     check_kind.as_ref(),
-                    check_kind.summary().replace("|", r"\|"),
+                    check_kind.summary().replace('|', r"\|"),
                     fix_token
                 ));
                 output.push('\n');

ruff_dev/src/generate_source_code.rs

@@ -1,25 +1,25 @@
+//! Run round-trip source code generation on a given Python file.
+
 use std::path::PathBuf;
 
 use anyhow::Result;
-use clap::Parser;
+use clap::Args;
 use ruff::code_gen::SourceGenerator;
 use ruff::fs;
 use rustpython_parser::parser;
 
-#[derive(Debug, Parser)]
+#[derive(Args)]
-struct Cli {
+pub struct Cli {
+    /// Python file to round-trip.
     #[arg(required = true)]
     file: PathBuf,
 }
 
-fn main() -> Result<()> {
+pub fn main(cli: &Cli) -> Result<()> {
-    let cli = Cli::parse();
-
     let contents = fs::read_file(&cli.file)?;
     let python_ast = parser::parse_program(&contents, &cli.file.to_string_lossy())?;
     let mut generator = SourceGenerator::new();
     generator.unparse_suite(&python_ast)?;
     println!("{}", generator.generate()?);
-
     Ok(())
 }

ruff_dev/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod generate_check_code_prefix;
+pub mod generate_rules_table;
+pub mod generate_source_code;
+pub mod print_ast;
+pub mod print_tokens;

ruff_dev/src/main.rs

@@ -0,0 +1,39 @@
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+use ruff_dev::{
+    generate_check_code_prefix, generate_rules_table, generate_source_code, print_ast, print_tokens,
+};
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+#[command(propagate_version = true)]
+struct Cli {
+    #[command(subcommand)]
+    command: Commands,
+}
+
+#[derive(Subcommand)]
+enum Commands {
+    /// Generate the `CheckCodePrefix` enum.
+    GenerateCheckCodePrefix(generate_check_code_prefix::Cli),
+    /// Generate a Markdown-compatible table of supported lint rules.
+    GenerateRulesTable(generate_rules_table::Cli),
+    /// Run round-trip source code generation on a given Python file.
+    GenerateSourceCode(generate_source_code::Cli),
+    /// Print the AST for a given Python file.
+    PrintAST(print_ast::Cli),
+    /// Print the token stream for a given Python file.
+    PrintTokens(print_tokens::Cli),
+}
+
+fn main() -> Result<()> {
+    let cli = Cli::parse();
+    match &cli.command {
+        Commands::GenerateCheckCodePrefix(args) => generate_check_code_prefix::main(args)?,
+        Commands::GenerateRulesTable(args) => generate_rules_table::main(args)?,
+        Commands::GenerateSourceCode(args) => generate_source_code::main(args)?,
+        Commands::PrintAST(args) => print_ast::main(args)?,
+        Commands::PrintTokens(args) => print_tokens::main(args)?,
+    }
+    Ok(())
+}

ruff_dev/src/print_ast.rs

@@ -3,23 +3,20 @@
 use std::path::PathBuf;
 
 use anyhow::Result;
-use clap::Parser;
+use clap::Args;
 use ruff::fs;
 use rustpython_parser::parser;
 
-#[derive(Debug, Parser)]
+#[derive(Args)]
-struct Cli {
+pub struct Cli {
+    /// Python file for which to generate the AST.
     #[arg(required = true)]
     file: PathBuf,
 }
 
-fn main() -> Result<()> {
+pub fn main(cli: &Cli) -> Result<()> {
-    let cli = Cli::parse();
-
     let contents = fs::read_file(&cli.file)?;
     let python_ast = parser::parse_program(&contents, &cli.file.to_string_lossy())?;
-
     println!("{:#?}", python_ast);
-
     Ok(())
 }

ruff_dev/src/print_tokens.rs

@@ -3,23 +3,21 @@
 use std::path::PathBuf;
 
 use anyhow::Result;
-use clap::Parser;
+use clap::Args;
 use ruff::fs;
 use rustpython_parser::lexer;
 
-#[derive(Debug, Parser)]
+#[derive(Args)]
-struct Cli {
+pub struct Cli {
+    /// Python file for which to generate the AST.
     #[arg(required = true)]
     file: PathBuf,
 }
 
-fn main() -> Result<()> {
+pub fn main(cli: &Cli) -> Result<()> {
-    let cli = Cli::parse();
-
     let contents = fs::read_file(&cli.file)?;
     for (_, tok, _) in lexer::make_tokenizer(&contents).flatten() {
         println!("{:#?}", tok);
     }
-
     Ok(())
 }