Use a faster diffing library for the formatter ecosystem checks (#6497)

**Summary** Some files seems notoriously slow in the formatter (secons in debug mode). This time was however almost exclusively spent in the diff algorithm to collect the similarity index, so i replaced that. I kept `similar` for printing actual diff to avoid rewriting that too, with the disadvantage that we now have to diff libraries in format_dev.

I used this PR to remove the spinner from tracing-indicatif and changed `flamegraph --perfdata perf.data` to `flamegraph --perfdata perf.data --no-inline` as the former wouldn't finish for me on release builds with debug info.

CONTRIBUTING.md

@@ -571,7 +571,7 @@ An alternative is to convert the perf data to `flamegraph.svg` using
 [flamegraph](https://github.com/flamegraph-rs/flamegraph) (`cargo install flamegraph`):
 
 ```shell
-flamegraph --perfdata perf.data
+flamegraph --perfdata perf.data --no-inline
 ```
 
 #### Mac

Cargo.lock

@@ -14,6 +14,18 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "ahash"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f"
+dependencies = [
+ "cfg-if",
+ "getrandom",
+ "once_cell",
+ "version_check",
+]
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.20"
@@ -991,6 +1003,16 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "imara-diff"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8"
+dependencies = [
+ "ahash",
+ "hashbrown 0.12.3",
+]
+
 [[package]]
 name = "imperative"
 version = "1.0.4"
@@ -2197,6 +2219,7 @@ dependencies = [
  "anyhow",
  "clap",
  "ignore",
+ "imara-diff",
  "indicatif",
  "indoc",
  "itertools",

crates/ruff_dev/Cargo.toml

@@ -44,6 +44,11 @@ toml = { workspace = true, features = ["parse"] }
 tracing = { workspace = true }
 tracing-indicatif = { workspace = true }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
+imara-diff = "0.1.5"
+
+[features]
+# Turn off rayon for profiling
+singlethreaded = []
 
 [dev-dependencies]
 indoc = "2.0.3"

crates/ruff_dev/src/format_dev.rs

@@ -1,7 +1,11 @@
 use anyhow::{bail, format_err, Context, Error};
 use clap::{CommandFactory, FromArgMatches};
 use ignore::DirEntry;
+use imara_diff::intern::InternedInput;
+use imara_diff::sink::Counter;
+use imara_diff::{diff, Algorithm};
 use indicatif::ProgressStyle;
+#[cfg_attr(feature = "singlethreaded", allow(unused_imports))]
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use ruff::logging::LogLevel;
 use ruff::resolver::python_files_in_path;
@@ -99,16 +103,18 @@ impl Statistics {
                 intersection,
             }
         } else {
-            let diff = TextDiff::from_lines(black, ruff);
-            let mut statistics = Self::default();
-            for change in diff.iter_all_changes() {
-                match change.tag() {
-                    ChangeTag::Delete => statistics.black_input += 1,
-                    ChangeTag::Insert => statistics.ruff_output += 1,
-                    ChangeTag::Equal => statistics.intersection += 1,
-                }
+            // `similar` was too slow (for some files >90% diffing instead of formatting)
+            let input = InternedInput::new(black, ruff);
+            let changes = diff(Algorithm::Histogram, &input, Counter::default());
+            assert_eq!(
+                input.before.len() - (changes.removals as usize),
+                input.after.len() - (changes.insertions as usize)
+            );
+            Self {
+                black_input: changes.removals,
+                ruff_output: changes.insertions,
+                intersection: u32::try_from(input.before.len()).unwrap() - changes.removals,
             }
-            statistics
         }
     }
 
@@ -253,7 +259,10 @@ fn setup_logging(log_level_args: &LogLevelArgs, log_file: Option<&Path>) -> io::
             .with_default_directive(log_level.into())
             .parse_lossy("")
     });
-    let indicatif_layer = IndicatifLayer::new();
+    let indicatif_layer = IndicatifLayer::new().with_progress_style(
+        // Default without the spinner
+        ProgressStyle::with_template("{span_child_prefix} {span_name}{{{span_fields}}}").unwrap(),
+    );
     let indicitif_compatible_writer_layer = tracing_subscriber::fmt::layer()
         .with_writer(indicatif_layer.get_stderr_writer())
         .with_target(false);
@@ -430,14 +439,16 @@ fn format_dev_project(
         pb_span.pb_set_style(&ProgressStyle::default_bar());
         pb_span.pb_set_length(paths.len() as u64);
         let _pb_span_enter = pb_span.enter();
-        paths
-            .into_par_iter()
-            .map(|dir_entry| {
-                let result = format_dir_entry(dir_entry, stability_check, write, &black_options);
-                pb_span.pb_inc(1);
-                result
-            })
-            .collect::<anyhow::Result<Vec<_>>>()?
+        #[cfg(not(feature = "singlethreaded"))]
+        let iter = { paths.into_par_iter() };
+        #[cfg(feature = "singlethreaded")]
+        let iter = { paths.into_iter() };
+        iter.map(|dir_entry| {
+            let result = format_dir_entry(dir_entry, stability_check, write, &black_options);
+            pb_span.pb_inc(1);
+            result
+        })
+        .collect::<anyhow::Result<Vec<_>>>()?
     };
 
     let mut statistics = Statistics::default();