Add Formatter benchmark (#4860)

.cargo/config.toml

@@ -1,6 +1,6 @@
 [alias]
 dev = "run --package ruff_dev --bin ruff_dev"
-benchmark = "bench -p ruff_benchmark --"
+benchmark = "bench -p ruff_benchmark --bench linter --bench formatter --"
 
 [target.'cfg(all())']
 rustflags = [

Cargo.lock

@@ -1796,6 +1796,7 @@ dependencies = [
  "once_cell",
  "ruff",
  "ruff_python_ast",
+ "ruff_python_formatter",
  "rustpython-parser",
  "serde",
  "serde_json",

crates/ruff_benchmark/Cargo.toml

@@ -20,6 +20,10 @@ harness = false
 name = "parser"
 harness = false
 
+[[bench]]
+name = "formatter"
+harness = false
+
 [dependencies]
 once_cell.workspace = true
 serde.workspace = true
@@ -30,6 +34,7 @@ ureq = "2.6.2"
 [dev-dependencies]
 ruff.path = "../ruff"
 ruff_python_ast.path = "../ruff_python_ast"
+ruff_python_formatter = { path = "../ruff_python_formatter" }
 criterion = { version = "0.5.1"}
 rustpython-parser.workspace = true
 
@@ -38,3 +43,4 @@ mimalloc = "0.1.34"
 
 [target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dev-dependencies]
 tikv-jemallocator = "0.5.0"
+

crates/ruff_benchmark/benches/formatter.rs

@@ -0,0 +1,62 @@
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
+use ruff_python_formatter::format_module;
+use std::time::Duration;
+
+#[cfg(target_os = "windows")]
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+#[cfg(all(
+    not(target_os = "windows"),
+    not(target_os = "openbsd"),
+    any(
+        target_arch = "x86_64",
+        target_arch = "aarch64",
+        target_arch = "powerpc64"
+    )
+))]
+#[global_allocator]
+static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
+
+fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
+    Ok(vec![
+        TestCase::fast(TestFile::try_download("numpy/globals.py", "https://raw.githubusercontent.com/numpy/numpy/89d64415e349ca75a25250f22b874aa16e5c0973/numpy/_globals.py")?),
+        TestCase::normal(TestFile::try_download(
+            "pydantic/types.py",
+            "https://raw.githubusercontent.com/pydantic/pydantic/83b3c49e99ceb4599d9286a3d793cea44ac36d4b/pydantic/types.py",
+        )?),
+        TestCase::normal(TestFile::try_download("numpy/ctypeslib.py", "https://raw.githubusercontent.com/numpy/numpy/e42c9503a14d66adfd41356ef5640c6975c45218/numpy/ctypeslib.py")?),
+        TestCase::slow(TestFile::try_download(
+            "large/dataset.py",
+            "https://raw.githubusercontent.com/DHI/mikeio/b7d26418f4db2909b0aa965253dbe83194d7bb5b/tests/test_dataset.py",
+        )?),
+    ])
+}
+
+fn benchmark_formatter(criterion: &mut Criterion) {
+    let mut group = criterion.benchmark_group("formatter");
+    let test_cases = create_test_cases().unwrap();
+
+    for case in test_cases {
+        group.throughput(Throughput::Bytes(case.code().len() as u64));
+        group.measurement_time(match case.speed() {
+            TestCaseSpeed::Fast => Duration::from_secs(5),
+            TestCaseSpeed::Normal => Duration::from_secs(10),
+            TestCaseSpeed::Slow => Duration::from_secs(20),
+        });
+
+        group.bench_with_input(
+            BenchmarkId::from_parameter(case.name()),
+            &case,
+            |b, case| {
+                b.iter(|| format_module(case.code()).expect("Formatting to succeed"));
+            },
+        );
+    }
+
+    group.finish();
+}
+
+criterion_group!(formatter, benchmark_formatter);
+criterion_main!(formatter);

crates/ruff_python_formatter/src/comments/debug.rs

@@ -183,7 +183,6 @@ mod tests {
     use ruff_python_ast::node::AnyNode;
     use ruff_text_size::{TextRange, TextSize};
     use rustpython_parser::ast::{StmtBreak, StmtContinue};
-    use std::cell::Cell;
 
     #[test]
     fn debug() {
@@ -210,7 +209,7 @@ break;
             SourceComment {
                 slice: source_code.slice(TextRange::at(TextSize::new(0), TextSize::new(17))),
                 #[cfg(debug_assertions)]
-                formatted: Cell::new(false),
+                formatted: std::cell::Cell::new(false),
                 position: CommentTextPosition::OwnLine,
             },
         );
@@ -220,7 +219,7 @@ break;
             SourceComment {
                 slice: source_code.slice(TextRange::at(TextSize::new(28), TextSize::new(10))),
                 #[cfg(debug_assertions)]
-                formatted: Cell::new(false),
+                formatted: std::cell::Cell::new(false),
                 position: CommentTextPosition::EndOfLine,
             },
         );
@@ -230,7 +229,7 @@ break;
             SourceComment {
                 slice: source_code.slice(TextRange::at(TextSize::new(39), TextSize::new(15))),
                 #[cfg(debug_assertions)]
-                formatted: Cell::new(false),
+                formatted: std::cell::Cell::new(false),
                 position: CommentTextPosition::OwnLine,
             },
         );

crates/ruff_python_formatter/src/comments/mod.rs

@@ -88,7 +88,6 @@
 //! It is possible to add an additional optional label to [`SourceComment`] If ever the need arises to distinguish two *dangling comments* in the formatting logic,
 
 use rustpython_parser::ast::Mod;
-use std::cell::Cell;
 use std::fmt::Debug;
 use std::rc::Rc;
 
@@ -119,7 +118,7 @@ pub(crate) struct SourceComment {
 
     /// Whether the comment has been formatted or not.
     #[cfg(debug_assertions)]
-    formatted: Cell<bool>,
+    formatted: std::cell::Cell<bool>,
 
     position: CommentTextPosition,
 }
@@ -137,7 +136,7 @@ impl SourceComment {
 
     #[cfg(not(debug_assertions))]
     #[inline(always)]
-    pub fn mark_formatted(&self) {}
+    pub(crate) fn mark_formatted(&self) {}
 
     /// Marks the comment as formatted
     #[cfg(debug_assertions)]

crates/ruff_python_formatter/src/comments/visitor.rs

@@ -5,7 +5,6 @@ use ruff_formatter::{SourceCode, SourceCodeSlice};
 use ruff_python_ast::node::AnyNodeRef;
 use ruff_python_ast::prelude::*;
 use ruff_python_ast::source_code::{CommentRanges, Locator};
-use std::cell::Cell;
 // The interface is designed to only export the members relevant for iterating nodes in
 // pre-order.
 #[allow(clippy::wildcard_imports)]
@@ -418,7 +417,7 @@ impl From<DecoratedComment<'_>> for SourceComment {
             slice: decorated.slice,
             position: decorated.text_position,
             #[cfg(debug_assertions)]
-            formatted: Cell::new(false),
+            formatted: std::cell::Cell::new(false),
         }
     }
 }