Rename ruff_python_whitespace to ruff_python_trivia (#5886)

## Summary

This crate now contains utilities for dealing with trivia more broadly:
whitespace, newlines, "simple" trivia lexing, etc. So renaming it to
reflect its increased responsibilities.

To avoid conflicts, I've also renamed `Token` and `TokenKind` to
`SimpleToken` and `SimpleTokenKind`.

crates/ruff_python_formatter/src/expression/expr_call.rs

@@ -3,7 +3,7 @@ use rustpython_parser::ast::{Expr, ExprCall, Ranged};
 
 use ruff_formatter::write;
 use ruff_python_ast::node::AnyNodeRef;
-use ruff_python_whitespace::{SimpleTokenizer, TokenKind};
+use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
 
 use crate::comments::dangling_comments;
 use crate::expression::expr_generator_exp::GeneratorExpParentheses;
@@ -132,14 +132,14 @@ fn is_single_argument_parenthesized(argument: &Expr, call_end: TextSize, source:
         SimpleTokenizer::new(source, TextRange::new(argument.end(), call_end)).skip_trivia()
     {
         match token.kind() {
-            TokenKind::RParen => {
+            SimpleTokenKind::RParen => {
                 if has_seen_r_paren {
                     return true;
                 }
                 has_seen_r_paren = true;
             }
             // Skip over any trailing comma
-            TokenKind::Comma => continue,
+            SimpleTokenKind::Comma => continue,
             _ => {
                 // Passed the arguments
                 break;

crates/ruff_python_formatter/src/expression/expr_slice.rs

@@ -5,7 +5,7 @@ use rustpython_parser::ast::{Expr, Ranged};
 use ruff_formatter::prelude::{hard_line_break, line_suffix_boundary, space, text};
 use ruff_formatter::{write, Buffer, Format, FormatError, FormatResult};
 use ruff_python_ast::node::{AnyNodeRef, AstNode};
-use ruff_python_whitespace::{SimpleTokenizer, Token, TokenKind};
+use ruff_python_trivia::{SimpleToken, SimpleTokenKind, SimpleTokenizer};
 
 use crate::comments::{dangling_comments, SourceComment};
 use crate::context::PyFormatContext;
@@ -158,17 +158,17 @@ pub(crate) fn find_colons(
     range: TextRange,
     lower: &Option<Box<Expr>>,
     upper: &Option<Box<Expr>>,
-) -> FormatResult<(Token, Option<Token>)> {
+) -> FormatResult<(SimpleToken, Option<SimpleToken>)> {
     let after_lower = lower
         .as_ref()
         .map_or(range.start(), |lower| lower.range().end());
     let mut tokens = SimpleTokenizer::new(contents, TextRange::new(after_lower, range.end()))
         .skip_trivia()
-        .skip_while(|token| token.kind == TokenKind::RParen);
+        .skip_while(|token| token.kind == SimpleTokenKind::RParen);
     let first_colon = tokens.next().ok_or(FormatError::syntax_error(
         "Din't find any token for slice first colon",
     ))?;
-    if first_colon.kind != TokenKind::Colon {
+    if first_colon.kind != SimpleTokenKind::Colon {
         return Err(FormatError::syntax_error(
             "slice first colon token was not a colon",
         ));
@@ -179,9 +179,9 @@ pub(crate) fn find_colons(
         .map_or(first_colon.end(), |upper| upper.range().end());
     let mut tokens = SimpleTokenizer::new(contents, TextRange::new(after_upper, range.end()))
         .skip_trivia()
-        .skip_while(|token| token.kind == TokenKind::RParen);
+        .skip_while(|token| token.kind == SimpleTokenKind::RParen);
     let second_colon = if let Some(token) = tokens.next() {
-        if token.kind != TokenKind::Colon {
+        if token.kind != SimpleTokenKind::Colon {
             return Err(FormatError::syntax_error(
                 "Expected a colon for the second colon token",
             ));

crates/ruff_python_formatter/src/expression/expr_unary_op.rs

@@ -5,7 +5,7 @@ use rustpython_parser::ast::{ExprUnaryOp, Ranged};
 use ruff_formatter::prelude::{hard_line_break, space, text};
 use ruff_formatter::{Format, FormatContext, FormatResult};
 use ruff_python_ast::node::AnyNodeRef;
-use ruff_python_whitespace::{SimpleTokenizer, TokenKind};
+use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
 
 use crate::comments::trailing_comments;
 use crate::context::PyFormatContext;
@@ -97,7 +97,7 @@ fn is_operand_parenthesized(unary: &ExprUnaryOp, source: &str) -> bool {
         .skip_trivia()
         .next()
     {
-        debug_assert_eq!(token.kind(), TokenKind::LParen);
+        debug_assert_eq!(token.kind(), SimpleTokenKind::LParen);
         true
     } else {
         false

crates/ruff_python_formatter/src/expression/parentheses.rs

@@ -3,7 +3,7 @@ use rustpython_parser::ast::Ranged;
 use ruff_formatter::prelude::tag::Condition;
 use ruff_formatter::{format_args, write, Argument, Arguments};
 use ruff_python_ast::node::AnyNodeRef;
-use ruff_python_whitespace::{first_non_trivia_token, SimpleTokenizer, Token, TokenKind};
+use ruff_python_trivia::{first_non_trivia_token, SimpleToken, SimpleTokenKind, SimpleTokenizer};
 
 use crate::context::NodeLevel;
 use crate::prelude::*;
@@ -77,8 +77,8 @@ pub(crate) fn is_expression_parenthesized(expr: AnyNodeRef, contents: &str) -> b
     // First test if there's a closing parentheses because it tends to be cheaper.
     if matches!(
         first_non_trivia_token(expr.end(), contents),
-        Some(Token {
-            kind: TokenKind::RParen,
+        Some(SimpleToken {
+            kind: SimpleTokenKind::RParen,
             ..
         })
     ) {
@@ -87,8 +87,8 @@ pub(crate) fn is_expression_parenthesized(expr: AnyNodeRef, contents: &str) -> b
 
         matches!(
             tokenizer.next_back(),
-            Some(Token {
-                kind: TokenKind::LParen,
+            Some(SimpleToken {
+                kind: SimpleTokenKind::LParen,
                 ..
             })
         )