refactor: Introduce crates folder (#2088)

This PR introduces a new `crates` directory and moves all "product" crates into that folder. 

Part of #2059.

crates/ruff_dev/src/generate_all.rs

@@ -0,0 +1,28 @@
+//! Run all code and documentation generation steps.
+
+use anyhow::Result;
+
+use crate::{generate_cli_help, generate_json_schema, generate_options, generate_rules_table};
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Write the generated artifacts to stdout (rather than to the filesystem).
+    #[arg(long)]
+    dry_run: bool,
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    generate_json_schema::main(&generate_json_schema::Args {
+        dry_run: args.dry_run,
+    })?;
+    generate_rules_table::main(&generate_rules_table::Args {
+        dry_run: args.dry_run,
+    })?;
+    generate_options::main(&generate_options::Args {
+        dry_run: args.dry_run,
+    })?;
+    generate_cli_help::main(&generate_cli_help::Args {
+        dry_run: args.dry_run,
+    })?;
+    Ok(())
+}

crates/ruff_dev/src/generate_cli_help.rs

@@ -0,0 +1,49 @@
+//! Generate CLI help.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use crate::utils::replace_readme_section;
+use anyhow::Result;
+use std::str;
+
+const COMMAND_HELP_BEGIN_PRAGMA: &str = "<!-- Begin auto-generated command help. -->\n";
+const COMMAND_HELP_END_PRAGMA: &str = "<!-- End auto-generated command help. -->";
+
+const SUBCOMMAND_HELP_BEGIN_PRAGMA: &str = "<!-- Begin auto-generated subcommand help. -->\n";
+const SUBCOMMAND_HELP_END_PRAGMA: &str = "<!-- End auto-generated subcommand help. -->";
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Write the generated help to stdout (rather than to `README.md`).
+    #[arg(long)]
+    pub(crate) dry_run: bool,
+}
+
+fn trim_lines(s: &str) -> String {
+    s.lines().map(str::trim_end).collect::<Vec<_>>().join("\n")
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    // Generate `ruff help`.
+    let command_help = trim_lines(ruff_cli::command_help().trim());
+
+    // Generate `ruff help check`.
+    let subcommand_help = trim_lines(ruff_cli::subcommand_help().trim());
+
+    if args.dry_run {
+        print!("{command_help}");
+        print!("{subcommand_help}");
+    } else {
+        replace_readme_section(
+            &format!("```text\n{command_help}\n```\n\n"),
+            COMMAND_HELP_BEGIN_PRAGMA,
+            COMMAND_HELP_END_PRAGMA,
+        )?;
+        replace_readme_section(
+            &format!("```text\n{subcommand_help}\n```\n\n"),
+            SUBCOMMAND_HELP_BEGIN_PRAGMA,
+            SUBCOMMAND_HELP_END_PRAGMA,
+        )?;
+    }
+
+    Ok(())
+}

crates/ruff_dev/src/generate_json_schema.rs

@@ -0,0 +1,30 @@
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use ruff::settings::options::Options;
+use schemars::schema_for;
+
+use crate::ROOT_DIR;
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Write the generated table to stdout (rather than to `ruff.schema.json`).
+    #[arg(long)]
+    pub(crate) dry_run: bool,
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    let schema = schema_for!(Options);
+    let schema_string = serde_json::to_string_pretty(&schema).unwrap();
+
+    if args.dry_run {
+        println!("{schema_string}");
+    } else {
+        let file = PathBuf::from(ROOT_DIR).join("ruff.schema.json");
+        fs::write(file, schema_string.as_bytes())?;
+    }
+    Ok(())
+}

crates/ruff_dev/src/generate_options.rs

@@ -0,0 +1,99 @@
+//! Generate a Markdown-compatible listing of configuration options.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use anyhow::Result;
+use itertools::Itertools;
+use ruff::settings::options::Options;
+use ruff::settings::options_base::{ConfigurationOptions, OptionEntry, OptionField};
+
+use crate::utils::replace_readme_section;
+
+const BEGIN_PRAGMA: &str = "<!-- Begin auto-generated options sections. -->\n";
+const END_PRAGMA: &str = "<!-- End auto-generated options sections. -->";
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Write the generated table to stdout (rather than to `README.md`).
+    #[arg(long)]
+    pub(crate) dry_run: bool,
+}
+
+fn emit_field(output: &mut String, field: &OptionField, group_name: Option<&str>) {
+    output.push_str(&format!("#### [`{0}`](#{0})\n", field.name));
+    output.push('\n');
+    output.push_str(field.doc);
+    output.push_str("\n\n");
+    output.push_str(&format!("**Default value**: `{}`\n", field.default));
+    output.push('\n');
+    output.push_str(&format!("**Type**: `{}`\n", field.value_type));
+    output.push('\n');
+    output.push_str(&format!(
+        "**Example usage**:\n\n```toml\n[tool.ruff{}]\n{}\n```\n",
+        if group_name.is_some() {
+            format!(".{}", group_name.unwrap())
+        } else {
+            String::new()
+        },
+        field.example
+    ));
+    output.push('\n');
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    let mut output = String::new();
+
+    // Generate all the top-level fields.
+    for field in Options::get_available_options()
+        .into_iter()
+        .filter_map(|entry| {
+            if let OptionEntry::Field(field) = entry {
+                Some(field)
+            } else {
+                None
+            }
+        })
+        .sorted_by_key(|field| field.name)
+    {
+        emit_field(&mut output, &field, None);
+        output.push_str("---\n\n");
+    }
+
+    // Generate all the sub-groups.
+    for group in Options::get_available_options()
+        .into_iter()
+        .filter_map(|entry| {
+            if let OptionEntry::Group(group) = entry {
+                Some(group)
+            } else {
+                None
+            }
+        })
+        .sorted_by_key(|group| group.name)
+    {
+        output.push_str(&format!("### `{}`\n", group.name));
+        output.push('\n');
+        for field in group
+            .fields
+            .iter()
+            .filter_map(|entry| {
+                if let OptionEntry::Field(field) = entry {
+                    Some(field)
+                } else {
+                    None
+                }
+            })
+            .sorted_by_key(|field| field.name)
+        {
+            emit_field(&mut output, field, Some(group.name));
+            output.push_str("---\n\n");
+        }
+    }
+
+    if args.dry_run {
+        print!("{output}");
+    } else {
+        replace_readme_section(&output, BEGIN_PRAGMA, END_PRAGMA)?;
+    }
+
+    Ok(())
+}

crates/ruff_dev/src/generate_rules_table.rs

@@ -0,0 +1,118 @@
+//! Generate a Markdown-compatible table of supported lint rules.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use anyhow::Result;
+use itertools::Itertools;
+use ruff::registry::{Linter, Rule, RuleNamespace, UpstreamCategory};
+use strum::IntoEnumIterator;
+
+use crate::utils::replace_readme_section;
+
+const TABLE_BEGIN_PRAGMA: &str = "<!-- Begin auto-generated sections. -->\n";
+const TABLE_END_PRAGMA: &str = "<!-- End auto-generated sections. -->";
+
+const TOC_BEGIN_PRAGMA: &str = "<!-- Begin auto-generated table of contents. -->";
+const TOC_END_PRAGMA: &str = "<!-- End auto-generated table of contents. -->";
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Write the generated table to stdout (rather than to `README.md`).
+    #[arg(long)]
+    pub(crate) dry_run: bool,
+}
+
+fn generate_table(table_out: &mut String, rules: impl IntoIterator<Item = Rule>) {
+    table_out.push_str("| Code | Name | Message | Fix |");
+    table_out.push('\n');
+    table_out.push_str("| ---- | ---- | ------- | --- |");
+    table_out.push('\n');
+    for rule in rules {
+        let fix_token = match rule.autofixable() {
+            None => "",
+            Some(_) => "🛠",
+        };
+
+        table_out.push_str(&format!(
+            "| {} | {} | {} | {} |",
+            rule.code(),
+            rule.as_ref(),
+            rule.message_formats()[0].replace('|', r"\|"),
+            fix_token
+        ));
+        table_out.push('\n');
+    }
+    table_out.push('\n');
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    // Generate the table string.
+    let mut table_out = String::new();
+    let mut toc_out = String::new();
+    for linter in Linter::iter() {
+        let codes_csv: String = match linter.common_prefix() {
+            "" => linter
+                .upstream_categories()
+                .unwrap()
+                .iter()
+                .map(|UpstreamCategory(prefix, ..)| prefix.as_ref())
+                .join(", "),
+            prefix => prefix.to_string(),
+        };
+        table_out.push_str(&format!("### {} ({codes_csv})", linter.name()));
+        table_out.push('\n');
+        table_out.push('\n');
+
+        toc_out.push_str(&format!(
+            "   1. [{} ({})](#{}-{})\n",
+            linter.name(),
+            codes_csv,
+            linter.name().to_lowercase().replace(' ', "-"),
+            codes_csv.to_lowercase().replace(',', "-").replace(' ', "")
+        ));
+
+        if let Some(url) = linter.url() {
+            let host = url
+                .trim_start_matches("https://")
+                .split('/')
+                .next()
+                .unwrap();
+            table_out.push_str(&format!(
+                "For more, see [{}]({}) on {}.",
+                linter.name(),
+                url,
+                match host {
+                    "pypi.org" => "PyPI",
+                    "github.com" => "GitHub",
+                    host => panic!(
+                        "unexpected host in URL of {}, expected pypi.org or github.com but found \
+                         {host}",
+                        linter.name()
+                    ),
+                }
+            ));
+            table_out.push('\n');
+            table_out.push('\n');
+        }
+
+        if let Some(categories) = linter.upstream_categories() {
+            for UpstreamCategory(prefix, name) in categories {
+                table_out.push_str(&format!("#### {name} ({})", prefix.as_ref()));
+                table_out.push('\n');
+                table_out.push('\n');
+                generate_table(&mut table_out, prefix);
+            }
+        } else {
+            generate_table(&mut table_out, &linter);
+        }
+    }
+
+    if args.dry_run {
+        print!("Table of Contents: {toc_out}\n Rules Tables: {table_out}");
+    } else {
+        // Extra newline in the markdown numbered list looks weird
+        replace_readme_section(toc_out.trim_end(), TOC_BEGIN_PRAGMA, TOC_END_PRAGMA)?;
+        replace_readme_section(&table_out, TABLE_BEGIN_PRAGMA, TABLE_END_PRAGMA)?;
+    }
+
+    Ok(())
+}

crates/ruff_dev/src/main.rs

@@ -0,0 +1,65 @@
+//! This crate implements an internal CLI for developers of Ruff.
+//!
+//! Within the ruff repository you can run it with `cargo dev`.
+
+mod generate_all;
+mod generate_cli_help;
+mod generate_json_schema;
+mod generate_options;
+mod generate_rules_table;
+mod print_ast;
+mod print_cst;
+mod print_tokens;
+mod round_trip;
+mod utils;
+
+use anyhow::Result;
+use clap::{Parser, Subcommand};
+
+const ROOT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../");
+
+#[derive(Parser)]
+#[command(author, version, about, long_about = None)]
+#[command(propagate_version = true)]
+struct Args {
+    #[command(subcommand)]
+    command: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+    /// Run all code and documentation generation steps.
+    GenerateAll(generate_all::Args),
+    /// Generate JSON schema for the TOML configuration file.
+    GenerateJSONSchema(generate_json_schema::Args),
+    /// Generate a Markdown-compatible table of supported lint rules.
+    GenerateRulesTable(generate_rules_table::Args),
+    /// Generate a Markdown-compatible listing of configuration options.
+    GenerateOptions(generate_options::Args),
+    /// Generate CLI help.
+    GenerateCliHelp(generate_cli_help::Args),
+    /// Print the AST for a given Python file.
+    PrintAST(print_ast::Args),
+    /// Print the LibCST CST for a given Python file.
+    PrintCST(print_cst::Args),
+    /// Print the token stream for a given Python file.
+    PrintTokens(print_tokens::Args),
+    /// Run round-trip source code generation on a given Python file.
+    RoundTrip(round_trip::Args),
+}
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+    match &args.command {
+        Command::GenerateAll(args) => generate_all::main(args)?,
+        Command::GenerateJSONSchema(args) => generate_json_schema::main(args)?,
+        Command::GenerateRulesTable(args) => generate_rules_table::main(args)?,
+        Command::GenerateOptions(args) => generate_options::main(args)?,
+        Command::GenerateCliHelp(args) => generate_cli_help::main(args)?,
+        Command::PrintAST(args) => print_ast::main(args)?,
+        Command::PrintCST(args) => print_cst::main(args)?,
+        Command::PrintTokens(args) => print_tokens::main(args)?,
+        Command::RoundTrip(args) => round_trip::main(args)?,
+    }
+    Ok(())
+}

crates/ruff_dev/src/print_ast.rs

@@ -0,0 +1,22 @@
+//! Print the AST for a given Python file.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use rustpython_parser::parser;
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Python file for which to generate the AST.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    let contents = fs::read_to_string(&args.file)?;
+    let python_ast = parser::parse_program(&contents, &args.file.to_string_lossy())?;
+    println!("{python_ast:#?}");
+    Ok(())
+}

crates/ruff_dev/src/print_cst.rs

@@ -0,0 +1,25 @@
+//! Print the `LibCST` CST for a given Python file.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::{bail, Result};
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Python file for which to generate the CST.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    let contents = fs::read_to_string(&args.file)?;
+    match libcst_native::parse_module(&contents, None) {
+        Ok(python_cst) => {
+            println!("{python_cst:#?}");
+            Ok(())
+        }
+        Err(_) => bail!("Failed to parse CST"),
+    }
+}

crates/ruff_dev/src/print_tokens.rs

@@ -0,0 +1,23 @@
+//! Print the token stream for a given Python file.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use rustpython_parser::lexer;
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Python file for which to generate the AST.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    let contents = fs::read_to_string(&args.file)?;
+    for (_, tok, _) in lexer::make_tokenizer(&contents).flatten() {
+        println!("{tok:#?}");
+    }
+    Ok(())
+}

crates/ruff_dev/src/round_trip.rs

@@ -0,0 +1,21 @@
+//! Run round-trip source code generation on a given Python file.
+#![allow(clippy::print_stdout, clippy::print_stderr)]
+
+use std::fs;
+use std::path::PathBuf;
+
+use anyhow::Result;
+use ruff::source_code::round_trip;
+
+#[derive(clap::Args)]
+pub struct Args {
+    /// Python file to round-trip.
+    #[arg(required = true)]
+    file: PathBuf,
+}
+
+pub fn main(args: &Args) -> Result<()> {
+    let contents = fs::read_to_string(&args.file)?;
+    println!("{}", round_trip(&contents, &args.file.to_string_lossy())?);
+    Ok(())
+}

crates/ruff_dev/src/utils.rs

@@ -0,0 +1,34 @@
+use std::fs;
+use std::fs::OpenOptions;
+use std::io::Write;
+use std::path::PathBuf;
+
+use anyhow::Result;
+
+use crate::ROOT_DIR;
+
+pub fn replace_readme_section(content: &str, begin_pragma: &str, end_pragma: &str) -> Result<()> {
+    // Read the existing file.
+    let file = PathBuf::from(ROOT_DIR).join("README.md");
+    let existing = fs::read_to_string(&file)?;
+
+    // Extract the prefix.
+    let index = existing
+        .find(begin_pragma)
+        .expect("Unable to find begin pragma");
+    let prefix = &existing[..index + begin_pragma.len()];
+
+    // Extract the suffix.
+    let index = existing
+        .find(end_pragma)
+        .expect("Unable to find end pragma");
+    let suffix = &existing[index..];
+
+    // Write the prefix, new contents, and suffix.
+    let mut f = OpenOptions::new().write(true).truncate(true).open(&file)?;
+    writeln!(f, "{prefix}")?;
+    write!(f, "{content}")?;
+    write!(f, "{suffix}")?;
+
+    Ok(())
+}