Add a ruff_textwrap crate (#4731)

Cargo.lock

@@ -14,17 +14,6 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
-[[package]]
-name = "ahash"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
-dependencies = [
- "getrandom",
- "once_cell",
- "version_check",
-]
-
 [[package]]
 name = "aho-corasick"
 version = "0.7.20"
@@ -809,9 +798,6 @@ name = "hashbrown"
 version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
-dependencies = [
- "ahash",
-]
 
 [[package]]
 name = "heck"
@@ -1798,6 +1784,7 @@ dependencies = [
  "ruff_python_stdlib",
  "ruff_rustpython",
  "ruff_text_size",
+ "ruff_textwrap",
  "rustc-hash",
  "rustpython-format",
  "rustpython-parser",
@@ -1811,7 +1798,6 @@ dependencies = [
  "strum",
  "strum_macros",
  "test-case",
- "textwrap",
  "thiserror",
  "toml",
  "typed-arena",
@@ -1880,13 +1866,13 @@ dependencies = [
  "ruff_python_ast",
  "ruff_python_stdlib",
  "ruff_text_size",
+ "ruff_textwrap",
  "rustc-hash",
  "serde",
  "serde_json",
  "shellexpand",
  "similar",
  "strum",
- "textwrap",
  "tikv-jemallocator",
  "ureq",
  "walkdir",
@@ -1907,13 +1893,13 @@ dependencies = [
  "ruff",
  "ruff_cli",
  "ruff_diagnostics",
+ "ruff_textwrap",
  "rustpython-format",
  "rustpython-parser",
  "schemars",
  "serde_json",
  "strum",
  "strum_macros",
- "textwrap",
 ]
 
 [[package]]
@@ -1956,8 +1942,8 @@ dependencies = [
  "itertools",
  "proc-macro2",
  "quote",
+ "ruff_textwrap",
  "syn 2.0.15",
- "textwrap",
 ]
 
 [[package]]
@@ -2066,6 +2052,14 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "ruff_textwrap"
+version = "0.0.0"
+dependencies = [
+ "ruff_newlines",
+ "ruff_text_size",
+]
+
 [[package]]
 name = "ruff_wasm"
 version = "0.0.0"
@@ -2357,12 +2351,6 @@ version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
-[[package]]
-name = "smawk"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043"
-
 [[package]]
 name = "spin"
 version = "0.5.2"
@@ -2506,11 +2494,6 @@ name = "textwrap"
 version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
-dependencies = [
- "smawk",
- "unicode-linebreak",
- "unicode-width",
-]
 
 [[package]]
 name = "thiserror"
@@ -2756,16 +2739,6 @@ version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
 
-[[package]]
-name = "unicode-linebreak"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137"
-dependencies = [
- "hashbrown",
- "regex",
-]
-
 [[package]]
 name = "unicode-normalization"
 version = "0.1.22"

Cargo.toml

@@ -49,7 +49,6 @@ strum = { version = "0.24.1", features = ["strum_macros"] }
 strum_macros = { version = "0.24.3" }
 syn = { version = "2.0.15" }
 test-case = { version = "3.0.0" }
-textwrap = { version = "0.16.0" }
 toml = { version = "0.7.2" }
 
 [profile.release]

crates/ruff/Cargo.toml

@@ -23,6 +23,7 @@ ruff_python_semantic = { path = "../ruff_python_semantic" }
 ruff_python_stdlib = { path = "../ruff_python_stdlib" }
 ruff_rustpython = { path = "../ruff_rustpython" }
 ruff_text_size = { workspace = true }
+ruff_textwrap = { path = "../ruff_textwrap" }
 
 annotate-snippets = { version = "0.9.1", features = ["color"] }
 anyhow = { workspace = true }
@@ -67,7 +68,6 @@ shellexpand = { workspace = true }
 smallvec = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
-textwrap = { workspace = true }
 thiserror = { version = "1.0.38" }
 toml = { workspace = true }
 typed-arena = { version = "2.0.2" }

crates/ruff/src/rules/isort/rules/organize_imports.rs

@@ -3,15 +3,16 @@ use std::path::Path;
 use itertools::{EitherOrBoth, Itertools};
 use ruff_text_size::TextRange;
 use rustpython_parser::ast::{Ranged, Stmt};
-use textwrap::indent;
 
 use ruff_diagnostics::{AutofixKind, Diagnostic, Edit, Fix, Violation};
 use ruff_macros::{derive_message_formats, violation};
+use ruff_newlines::StrExt;
 use ruff_python_ast::helpers::{
     followed_by_multi_statement_line, preceded_by_multi_statement_line, trailing_lines_end,
 };
 use ruff_python_ast::source_code::{Indexer, Locator, Stylist};
 use ruff_python_ast::whitespace::leading_space;
+use ruff_textwrap::indent;
 
 use crate::line_width::LineWidth;
 use crate::registry::AsRule;
@@ -69,8 +70,8 @@ fn extract_indentation_range(body: &[&Stmt], locator: &Locator) -> TextRange {
 /// Compares two strings, returning true if they are equal modulo whitespace
 /// at the start of each line.
 fn matches_ignoring_indentation(val1: &str, val2: &str) -> bool {
-    val1.lines()
+    val1.universal_newlines()
-        .zip_longest(val2.lines())
+        .zip_longest(val2.universal_newlines())
         .all(|pair| match pair {
             EitherOrBoth::Both(line1, line2) => line1.trim_start() == line2.trim_start(),
             _ => false,
@@ -153,7 +154,7 @@ pub(crate) fn organize_imports(
     let mut diagnostic = Diagnostic::new(UnsortedImports, range);
     if settings.rules.should_fix(diagnostic.kind.rule()) {
         diagnostic.set_fix(Fix::automatic(Edit::range_replacement(
-            indent(&expected, indentation),
+            indent(&expected, indentation).to_string(),
             range,
         )));
     }

crates/ruff/src/rules/pydocstyle/rules/sections.rs

@@ -13,6 +13,7 @@ use ruff_python_ast::helpers::identifier_range;
 use ruff_python_ast::{cast, whitespace};
 use ruff_python_semantic::analyze::visibility::is_staticmethod;
 use ruff_python_semantic::definition::{Definition, Member, MemberKind};
+use ruff_textwrap::dedent;
 
 use crate::checkers::ast::Checker;
 use crate::docstrings::sections::{SectionContext, SectionContexts, SectionKind};
@@ -780,7 +781,7 @@ fn args_section(context: &SectionContext) -> FxHashSet<String> {
         .map(|l| l.as_str())
         .filter(|line| line.starts_with(leading_space) || line.is_empty())
         .join("\n");
-    let args_content = textwrap::dedent(&relevant_lines);
+    let args_content = dedent(&relevant_lines);
 
     // Reformat each section.
     let mut args_sections: Vec<String> = vec![];

crates/ruff/src/rules/pyflakes/mod.rs

@@ -10,9 +10,9 @@ mod tests {
 
     use anyhow::Result;
     use regex::Regex;
+    use ruff_textwrap::dedent;
     use rustpython_parser::lexer::LexResult;
     use test_case::test_case;
-    use textwrap::dedent;
 
     use ruff_diagnostics::Diagnostic;
     use ruff_python_ast::source_code::{Indexer, Locator, Stylist};

crates/ruff/src/test.rs

@@ -1,13 +1,13 @@
 #![cfg(test)]
+//! Helper functions for the tests of rule implementations.
 
-/// Helper functions for the tests of rule implementations.
 use std::path::Path;
 
 use anyhow::Result;
 use itertools::Itertools;
+use ruff_textwrap::dedent;
 use rustc_hash::FxHashMap;
 use rustpython_parser::lexer::LexResult;
-use textwrap::dedent;
 
 use ruff_diagnostics::{AutofixKind, Diagnostic};
 use ruff_python_ast::source_code::{Indexer, Locator, SourceFileBuilder, Stylist};

crates/ruff_cli/Cargo.toml

@@ -27,6 +27,7 @@ ruff_cache = { path = "../ruff_cache" }
 ruff_diagnostics = { path = "../ruff_diagnostics" }
 ruff_python_ast = { path = "../ruff_python_ast" }
 ruff_text_size = { workspace = true }
+ruff_textwrap = { path = "../ruff_textwrap" }
 
 annotate-snippets = { version = "0.9.1", features = ["color"] }
 anyhow = { workspace = true }
@@ -56,7 +57,6 @@ serde_json = { workspace = true }
 shellexpand = { workspace = true }
 similar = { workspace = true }
 strum = { workspace = true, features = [] }
-textwrap = { workspace = true }
 walkdir = { version = "2.3.2" }
 wild = { version = "2" }
 

crates/ruff_dev/Cargo.toml

@@ -9,6 +9,7 @@ rust-version = { workspace = true }
 ruff = { path = "../ruff", features = ["schemars"] }
 ruff_cli = { path = "../ruff_cli" }
 ruff_diagnostics = { path = "../ruff_diagnostics" }
+ruff_textwrap = { path = "../ruff_textwrap" }
 
 anyhow = { workspace = true }
 clap = { workspace = true }
@@ -23,4 +24,3 @@ schemars = { workspace = true }
 serde_json = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
-textwrap = { workspace = true }

crates/ruff_macros/Cargo.toml

@@ -10,8 +10,9 @@ proc-macro = true
 doctest = false
 
 [dependencies]
+ruff_textwrap = { path = "../ruff_textwrap" }
+
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
 syn = { workspace = true, features = ["derive", "parsing", "extra-traits", "full"] }
-textwrap = { workspace = true }
 itertools = { workspace = true }

crates/ruff_macros/src/config.rs

@@ -1,3 +1,5 @@
+use ruff_textwrap::dedent;
+
 use quote::{quote, quote_spanned};
 use syn::parse::{Parse, ParseStream};
 use syn::spanned::Spanned;
@@ -126,7 +128,7 @@ fn handle_option(
     docs: Vec<&Attribute>,
 ) -> syn::Result<proc_macro2::TokenStream> {
     // Convert the list of `doc` attributes into a single string.
-    let doc = textwrap::dedent(
+    let doc = dedent(
         &docs
             .into_iter()
             .map(parse_doc)
@@ -179,7 +181,7 @@ impl Parse for FieldAttributes {
         Ok(Self {
             default,
             value_type,
-            example: textwrap::dedent(&example).trim_matches('\n').to_string(),
+            example: dedent(&example).trim_matches('\n').to_string(),
         })
     }
 }

crates/ruff_newlines/src/lib.rs

@@ -232,23 +232,29 @@ impl<'a> Line<'a> {
         TextRange::new(self.start(), self.end())
     }
 
+    /// Returns the line's new line character, if any.
+    #[inline]
+    pub fn line_ending(&self) -> Option<LineEnding> {
+        let mut bytes = self.text.bytes().rev();
+        match bytes.next() {
+            Some(b'\n') => {
+                if bytes.next() == Some(b'\r') {
+                    Some(LineEnding::CrLf)
+                } else {
+                    Some(LineEnding::Lf)
+                }
+            }
+            Some(b'\r') => Some(LineEnding::Cr),
+            _ => None,
+        }
+    }
+
     /// Returns the text of the line, excluding the terminating new line character.
     #[inline]
     pub fn as_str(&self) -> &'a str {
-        let mut bytes = self.text.bytes().rev();
+        let newline_len = self
-
+            .line_ending()
-        let newline_len = match bytes.next() {
+            .map_or(0, |line_ending| line_ending.len());
-            Some(b'\n') => {
-                if bytes.next() == Some(b'\r') {
-                    2
-                } else {
-                    1
-                }
-            }
-            Some(b'\r') => 1,
-            _ => 0,
-        };
-
         &self.text[..self.text.len() - newline_len]
     }
 

crates/ruff_textwrap/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "ruff_textwrap"
+version = "0.0.0"
+publish = false
+edition = { workspace = true }
+rust-version = { workspace = true }
+
+[dependencies]
+ruff_newlines = { path = "../ruff_newlines" }
+ruff_text_size = { workspace = true }

crates/ruff_textwrap/src/lib.rs

@@ -0,0 +1,336 @@
+//! Functions related to adding and removing indentation from lines of
+//! text.
+
+use std::borrow::Cow;
+use std::cmp;
+
+use ruff_newlines::StrExt;
+
+/// Indent each line by the given prefix.
+///
+/// # Examples
+///
+/// ```
+/// use ruff_textwrap::indent;
+///
+/// assert_eq!(indent("First line.\nSecond line.\n", "  "),
+///            "  First line.\n  Second line.\n");
+/// ```
+///
+/// When indenting, trailing whitespace is stripped from the prefix.
+/// This means that empty lines remain empty afterwards:
+///
+/// ```
+/// use ruff_textwrap::indent;
+///
+/// assert_eq!(indent("First line.\n\n\nSecond line.\n", "  "),
+///            "  First line.\n\n\n  Second line.\n");
+/// ```
+///
+/// Notice how `"\n\n\n"` remained as `"\n\n\n"`.
+///
+/// This feature is useful when you want to indent text and have a
+/// space between your prefix and the text. In this case, you _don't_
+/// want a trailing space on empty lines:
+///
+/// ```
+/// use ruff_textwrap::indent;
+///
+/// assert_eq!(indent("foo = 123\n\nprint(foo)\n", "# "),
+///            "# foo = 123\n#\n# print(foo)\n");
+/// ```
+///
+/// Notice how `"\n\n"` became `"\n#\n"` instead of `"\n# \n"` which
+/// would have trailing whitespace.
+///
+/// Leading and trailing whitespace coming from the text itself is
+/// kept unchanged:
+///
+/// ```
+/// use ruff_textwrap::indent;
+///
+/// assert_eq!(indent(" \t  Foo   ", "->"), "-> \t  Foo   ");
+/// ```
+pub fn indent<'a>(text: &'a str, prefix: &str) -> Cow<'a, str> {
+    if prefix.is_empty() {
+        return Cow::Borrowed(text);
+    }
+
+    let mut result = String::with_capacity(text.len() + prefix.len());
+    let trimmed_prefix = prefix.trim_end();
+    for line in text.universal_newlines() {
+        if line.trim().is_empty() {
+            result.push_str(trimmed_prefix);
+        } else {
+            result.push_str(prefix);
+        }
+        result.push_str(line.as_full_str());
+    }
+    Cow::Owned(result)
+}
+
+/// Removes common leading whitespace from each line.
+///
+/// This function will look at each non-empty line and determine the
+/// maximum amount of whitespace that can be removed from all lines:
+///
+/// ```
+/// use ruff_textwrap::dedent;
+///
+/// assert_eq!(dedent("
+///     1st line
+///       2nd line
+///     3rd line
+/// "), "
+/// 1st line
+///   2nd line
+/// 3rd line
+/// ");
+/// ```
+pub fn dedent(text: &str) -> Cow<'_, str> {
+    // Find the minimum amount of leading whitespace on each line.
+    let prefix_len = text
+        .universal_newlines()
+        .fold(usize::MAX, |prefix_len, line| {
+            let leading_whitespace_len = line.len() - line.trim_start().len();
+            if leading_whitespace_len == line.len() {
+                // Skip empty lines.
+                prefix_len
+            } else {
+                cmp::min(prefix_len, leading_whitespace_len)
+            }
+        });
+
+    // If there is no common prefix, no need to dedent.
+    if prefix_len == usize::MAX {
+        return Cow::Borrowed(text);
+    }
+
+    // Remove the common prefix from each line.
+    let mut result = String::with_capacity(text.len());
+    for line in text.universal_newlines() {
+        if line.trim().is_empty() {
+            if let Some(line_ending) = line.line_ending() {
+                result.push_str(&line_ending);
+            }
+        } else {
+            result.push_str(&line.as_full_str()[prefix_len..]);
+        }
+    }
+    Cow::Owned(result)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn indent_empty() {
+        assert_eq!(indent("\n", "  "), "\n");
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn indent_nonempty() {
+        let text = [
+            "  foo\n",
+            "bar\n",
+            "  baz\n",
+        ].join("");
+        let expected = [
+            "//   foo\n",
+            "// bar\n",
+            "//   baz\n",
+        ].join("");
+        assert_eq!(indent(&text, "// "), expected);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn indent_empty_line() {
+        let text = [
+            "  foo",
+            "bar",
+            "",
+            "  baz",
+        ].join("\n");
+        let expected = [
+            "//   foo",
+            "// bar",
+            "//",
+            "//   baz",
+        ].join("\n");
+        assert_eq!(indent(&text, "// "), expected);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn indent_mixed_newlines() {
+        let text = [
+            "  foo\r\n",
+            "bar\n",
+            "  baz\r",
+        ].join("");
+        let expected = [
+            "//   foo\r\n",
+            "// bar\n",
+            "//   baz\r",
+        ].join("");
+        assert_eq!(indent(&text, "// "), expected);
+    }
+
+    #[test]
+    fn dedent_empty() {
+        assert_eq!(dedent(""), "");
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_multi_line() {
+        let x = [
+            "    foo",
+            "  bar",
+            "    baz",
+        ].join("\n");
+        let y = [
+            "  foo",
+            "bar",
+            "  baz"
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_empty_line() {
+        let x = [
+            "    foo",
+            "  bar",
+            "   ",
+            "    baz"
+        ].join("\n");
+        let y = [
+            "  foo",
+            "bar",
+            "",
+            "  baz"
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_blank_line() {
+        let x = [
+            "      foo",
+            "",
+            "        bar",
+            "          foo",
+            "          bar",
+            "          baz",
+        ].join("\n");
+        let y = [
+            "foo",
+            "",
+            "  bar",
+            "    foo",
+            "    bar",
+            "    baz",
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_whitespace_line() {
+        let x = [
+            "      foo",
+            " ",
+            "        bar",
+            "          foo",
+            "          bar",
+            "          baz",
+        ].join("\n");
+        let y = [
+            "foo",
+            "",
+            "  bar",
+            "    foo",
+            "    bar",
+            "    baz",
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_mixed_whitespace() {
+        let x = [
+            "\tfoo",
+            "  bar",
+        ].join("\n");
+        let y = [
+            "foo",
+            " bar",
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_tabbed_whitespace() {
+        let x = [
+            "\t\tfoo",
+            "\t\t\tbar",
+        ].join("\n");
+        let y = [
+            "foo",
+            "\tbar",
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_mixed_tabbed_whitespace() {
+        let x = [
+            "\t  \tfoo",
+            "\t  \t\tbar",
+        ].join("\n");
+        let y = [
+            "foo",
+            "\tbar",
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_preserve_no_terminating_newline() {
+        let x = [
+            "  foo",
+            "    bar",
+        ].join("\n");
+        let y = [
+            "foo",
+            "  bar",
+        ].join("\n");
+        assert_eq!(dedent(&x), y);
+    }
+
+    #[test]
+    #[rustfmt::skip]
+    fn dedent_mixed_newlines() {
+        let x = [
+            "    foo\r\n",
+            "  bar\n",
+            "    baz\r",
+        ].join("");
+        let y = [
+            "  foo\r\n",
+            "bar\n",
+            "  baz\r"
+        ].join("");
+        assert_eq!(dedent(&x), y);
+    }
+}