Move Python whitespace utilities into new ruff_python_whitespace crate (#4993)

## Summary

`ruff_newlines` becomes `ruff_python_whitespace`, and includes the
existing "universal newline" handlers alongside the Python
whitespace-specific utilities.

crates/ruff_python_ast/Cargo.toml

@@ -8,7 +8,7 @@ rust-version = { workspace = true }
 [lib]
 
 [dependencies]
-ruff_newlines = { path = "../ruff_newlines" }
+ruff_python_whitespace = { path = "../ruff_python_whitespace" }
 ruff_text_size = { workspace = true }
 
 anyhow = { workspace = true }

crates/ruff_python_ast/src/docstrings.rs

@@ -0,0 +1,22 @@
+//! Utilities for parsing Python docstrings.
+
+/// Extract the leading words from a line of text within a Python docstring.
+pub fn leading_words(line: &str) -> &str {
+    let line = line.trim();
+    line.find(|char: char| !char.is_alphanumeric() && !char.is_whitespace())
+        .map_or(line, |index| &line[..index])
+}
+
+/// Extract the leading whitespace from a line of text within a Python docstring.
+pub fn leading_space(line: &str) -> &str {
+    line.find(|char: char| !char.is_whitespace())
+        .map_or(line, |index| &line[..index])
+}
+
+/// Replace any non-whitespace characters from an indentation string within a Python docstring.
+pub fn clean_space(indentation: &str) -> String {
+    indentation
+        .chars()
+        .map(|char| if char.is_whitespace() { char } else { ' ' })
+        .collect()
+}

crates/ruff_python_ast/src/helpers.rs

@@ -4,7 +4,7 @@ use std::path::Path;
 use itertools::Itertools;
 use log::error;
 use num_traits::Zero;
-use ruff_newlines::UniversalNewlineIterator;
+use ruff_python_whitespace::UniversalNewlineIterator;
 use ruff_text_size::{TextRange, TextSize};
 use rustc_hash::{FxHashMap, FxHashSet};
 use rustpython_parser::ast::{

crates/ruff_python_ast/src/lib.rs

@@ -2,6 +2,7 @@ pub mod all;
 pub mod call_path;
 pub mod cast;
 pub mod comparable;
+pub mod docstrings;
 pub mod function;
 pub mod hashable;
 pub mod helpers;

crates/ruff_python_ast/src/source_code/generator.rs

@@ -8,7 +8,7 @@ use rustpython_parser::ast::{
     Excepthandler, Expr, Identifier, MatchCase, Operator, Pattern, Stmt, Suite, Withitem,
 };
 
-use ruff_newlines::LineEnding;
+use ruff_python_whitespace::LineEnding;
 
 use crate::source_code::stylist::{Indentation, Quote, Stylist};
 
@@ -1459,7 +1459,7 @@ mod tests {
     use rustpython_ast::Suite;
     use rustpython_parser::Parse;
 
-    use ruff_newlines::LineEnding;
+    use ruff_python_whitespace::LineEnding;
 
     use crate::source_code::stylist::{Indentation, Quote};
     use crate::source_code::Generator;

crates/ruff_python_ast/src/source_code/locator.rs

@@ -6,7 +6,7 @@ use memchr::{memchr2, memrchr2};
 use once_cell::unsync::OnceCell;
 use ruff_text_size::{TextLen, TextRange, TextSize};
 
-use ruff_newlines::find_newline;
+use ruff_python_whitespace::find_newline;
 
 use crate::source_code::{LineIndex, OneIndexed, SourceCode, SourceLocation};
 

crates/ruff_python_ast/src/source_code/stylist.rs

@@ -4,7 +4,7 @@ use std::fmt;
 use std::ops::Deref;
 
 use once_cell::unsync::OnceCell;
-use ruff_newlines::{find_newline, LineEnding};
+use ruff_python_whitespace::{find_newline, LineEnding};
 use rustpython_literal::escape::Quote as StrQuote;
 use rustpython_parser::lexer::LexResult;
 use rustpython_parser::Tok;
@@ -166,7 +166,7 @@ mod tests {
     use rustpython_parser::lexer::lex;
     use rustpython_parser::Mode;
 
-    use ruff_newlines::{find_newline, LineEnding};
+    use ruff_python_whitespace::{find_newline, LineEnding};
 
     use crate::source_code::stylist::{Indentation, Quote};
     use crate::source_code::{Locator, Stylist};

crates/ruff_python_ast/src/whitespace.rs

@@ -1,6 +1,8 @@
 use ruff_text_size::{TextRange, TextSize};
 use rustpython_parser::ast::Ranged;
 
+use ruff_python_whitespace::is_python_whitespace;
+
 use crate::source_code::Locator;
 
 /// Extract the leading indentation from a line.
@@ -17,41 +19,9 @@ pub fn indentation_at_offset<'a>(locator: &'a Locator, offset: TextSize) -> Opti
     let line_start = locator.line_start(offset);
     let indentation = &locator.contents()[TextRange::new(line_start, offset)];
 
-    if indentation.chars().all(char::is_whitespace) {
+    if indentation.chars().all(is_python_whitespace) {
         Some(indentation)
     } else {
         None
     }
 }
-
-/// Extract the leading words from a line of text.
-pub fn leading_words(line: &str) -> &str {
-    let line = line.trim();
-    line.find(|char: char| !char.is_alphanumeric() && !char.is_whitespace())
-        .map_or(line, |index| &line[..index])
-}
-
-/// Extract the leading whitespace from a line of text.
-pub fn leading_space(line: &str) -> &str {
-    line.find(|char: char| !char.is_whitespace())
-        .map_or(line, |index| &line[..index])
-}
-
-/// Replace any non-whitespace characters from an indentation string.
-pub fn clean(indentation: &str) -> String {
-    indentation
-        .chars()
-        .map(|char| if char.is_whitespace() { char } else { ' ' })
-        .collect()
-}
-
-/// Returns `true` for [whitespace](https://docs.python.org/3/reference/lexical_analysis.html#whitespace-between-tokens)
-/// or new-line characters.
-pub const fn is_python_whitespace(c: char) -> bool {
-    matches!(
-        c,
-        ' ' | '\n' | '\t' | '\r' |
-        // Form-feed
-        '\x0C'
-    )
-}