Fix parenthesized detection for tuples (#6599)

## Summary

This PR fixes our code for detecting whether a tuple has its own
parentheses, which is necessary when attempting to preserve parentheses.
As-is, we were getting some cases wrong, like `(a := 1), (b := 3))` --
the detection code inferred that this _was_ parenthesized, and so
wrapped the entire thing in an unnecessary set of parentheses.

## Test Plan

`cargo test`

Before:

| project      | similarity index |
|--------------|------------------|
| cpython      | 0.75472          |
| django       | 0.99804          |
| transformers | 0.99618          |
| twine        | 0.99876          |
| typeshed     | 0.74288          |
| warehouse    | 0.99601          |
| zulip        | 0.99727          |

After:
| project      | similarity index |
|--------------|------------------|
| cpython      | 0.75473          |
| django       | 0.99804 |
| transformers | 0.99618          |
| twine        | 0.99876          |
| typeshed     | 0.74288          |
| warehouse    | 0.99601          |
| zulip        | 0.99727          |

crates/ruff_python_formatter/resources/test/fixtures/ruff/expression/tuple.py

@@ -3,6 +3,8 @@ a1 = 1, 2
 a2 = (1, 2)
 a3 = (1, 2), 3
 a4 = ((1, 2), 3)
+a5 = (1), (2)
+a6 = ((1), (2))
 
 # Wrapping parentheses checks
 b1 = (("Michael", "Ende"), ("Der", "satanarchäolügenialkohöllische", "Wunschpunsch"), ("Beelzebub", "Irrwitzer"), ("Tyrannja", "Vamperl"),)

crates/ruff_python_formatter/resources/test/fixtures/ruff/statement/for.py

@@ -32,3 +32,13 @@ for (x, y) in (z, w):
 # type comment
 for x in (): # type: int
     ...
+
+# Tuple parentheses for iterable.
+for x in 1, 2, 3:
+    pass
+
+for x in (1, 2, 3):
+    pass
+
+for x in 1, 2, 3,:
+    pass

crates/ruff_python_formatter/src/expression/expr_tuple.rs

@@ -1,7 +1,8 @@
 use ruff_formatter::{format_args, write, FormatRuleWithOptions};
 use ruff_python_ast::node::AnyNodeRef;
 use ruff_python_ast::ExprTuple;
-use ruff_python_ast::{Expr, Ranged};
+use ruff_python_ast::Ranged;
+use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
 use ruff_text_size::TextRange;
 
 use crate::builders::parenthesize_if_expands;
@@ -11,7 +12,7 @@ use crate::expression::parentheses::{
 };
 use crate::prelude::*;
 
-#[derive(Eq, PartialEq, Debug, Default)]
+#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)]
 pub enum TupleParentheses {
     /// By default tuples with a single element will include parentheses. Tuples with multiple elements
     /// will parenthesize if the expression expands. This means that tuples will often *preserve*
@@ -100,9 +101,9 @@ impl FormatRuleWithOptions<ExprTuple, PyFormatContext<'_>> for FormatExprTuple {
 impl FormatNodeRule<ExprTuple> for FormatExprTuple {
     fn fmt_fields(&self, item: &ExprTuple, f: &mut PyFormatter) -> FormatResult<()> {
         let ExprTuple {
-            range,
             elts,
             ctx: _,
+            range: _,
         } = item;
 
         let comments = f.context().comments().clone();
@@ -124,7 +125,7 @@ impl FormatNodeRule<ExprTuple> for FormatExprTuple {
             }
             [single] => match self.parentheses {
                 TupleParentheses::Preserve
-                    if !is_parenthesized(*range, elts, f.context().source()) =>
+                    if !is_tuple_parenthesized(item, f.context().source()) =>
                 {
                     write!(f, [single.format(), text(",")])
                 }
@@ -141,7 +142,7 @@ impl FormatNodeRule<ExprTuple> for FormatExprTuple {
             //
             // Unlike other expression parentheses, tuple parentheses are part of the range of the
             // tuple itself.
-            _ if is_parenthesized(*range, elts, f.context().source())
+            _ if is_tuple_parenthesized(item, f.context().source())
                 && !(self.parentheses == TupleParentheses::NeverPreserve
                     && dangling.is_empty()) =>
             {
@@ -203,21 +204,30 @@ impl NeedsParentheses for ExprTuple {
 }
 
 /// Check if a tuple has already had parentheses in the input
-fn is_parenthesized(tuple_range: TextRange, elts: &[Expr], source: &str) -> bool {
+fn is_tuple_parenthesized(tuple: &ExprTuple, source: &str) -> bool {
-    let parentheses = '(';
+    let Some(elt) = tuple.elts.first() else {
-    let first_char = &source[usize::from(tuple_range.start())..].chars().next();
-    let Some(first_char) = first_char else {
         return false;
     };
-    if *first_char != parentheses {
+
+    // Count the number of open parentheses between the start of the tuple and the first element.
+    let open_parentheses_count =
+        SimpleTokenizer::new(source, TextRange::new(tuple.start(), elt.start()))
+            .skip_trivia()
+            .filter(|token| token.kind() == SimpleTokenKind::LParen)
+            .count();
+    if open_parentheses_count == 0 {
         return false;
     }
 
-    // Consider `a = (1, 2), 3`: The first char of the current expr starts is a parentheses, but
+    // Count the number of parentheses between the end of the first element and its trailing comma.
-    // it's not its own but that of its first tuple child. We know that it belongs to the child
+    let close_parentheses_count =
-    // because if it wouldn't, the child would start (at least) a char later
+        SimpleTokenizer::new(source, TextRange::new(elt.end(), tuple.end()))
-    let Some(first_child) = elts.first() else {
+            .skip_trivia()
-        return false;
+            .take_while(|token| token.kind() != SimpleTokenKind::Comma)
-    };
+            .filter(|token| token.kind() == SimpleTokenKind::RParen)
-    first_child.range().start() != tuple_range.start()
+            .count();
+
+    // If the number of open parentheses is greater than the number of close parentheses, the tuple
+    // is parenthesized.
+    open_parentheses_count > close_parentheses_count
 }

crates/ruff_python_formatter/tests/snapshots/black_compatibility@py_39__pep_572_py39.py.snap

@@ -1,54 +0,0 @@
----
-source: crates/ruff_python_formatter/tests/fixtures.rs
-input_file: crates/ruff_python_formatter/resources/test/fixtures/black/py_39/pep_572_py39.py
----
-## Input
-
-```py
-# Unparenthesized walruses are now allowed in set literals & set comprehensions
-# since Python 3.9
-{x := 1, 2, 3}
-{x4 := x**5 for x in range(7)}
-# We better not remove the parentheses here (since it's a 3.10 feature)
-x[(a := 1)]
-x[(a := 1), (b := 3)]
-```
-
-## Black Differences
-
-```diff
---- Black
-+++ Ruff
-@@ -4,4 +4,4 @@
- {x4 := x**5 for x in range(7)}
- # We better not remove the parentheses here (since it's a 3.10 feature)
- x[(a := 1)]
--x[(a := 1), (b := 3)]
-+x[((a := 1), (b := 3))]
-```
-
-## Ruff Output
-
-```py
-# Unparenthesized walruses are now allowed in set literals & set comprehensions
-# since Python 3.9
-{x := 1, 2, 3}
-{x4 := x**5 for x in range(7)}
-# We better not remove the parentheses here (since it's a 3.10 feature)
-x[(a := 1)]
-x[((a := 1), (b := 3))]
-```
-
-## Black Output
-
-```py
-# Unparenthesized walruses are now allowed in set literals & set comprehensions
-# since Python 3.9
-{x := 1, 2, 3}
-{x4 := x**5 for x in range(7)}
-# We better not remove the parentheses here (since it's a 3.10 feature)
-x[(a := 1)]
-x[(a := 1), (b := 3)]
-```
-
-

crates/ruff_python_formatter/tests/snapshots/format@expression__tuple.py.snap

@@ -9,6 +9,8 @@ a1 = 1, 2
 a2 = (1, 2)
 a3 = (1, 2), 3
 a4 = ((1, 2), 3)
+a5 = (1), (2)
+a6 = ((1), (2))
 
 # Wrapping parentheses checks
 b1 = (("Michael", "Ende"), ("Der", "satanarchäolügenialkohöllische", "Wunschpunsch"), ("Beelzebub", "Irrwitzer"), ("Tyrannja", "Vamperl"),)
@@ -79,6 +81,8 @@ a1 = 1, 2
 a2 = (1, 2)
 a3 = (1, 2), 3
 a4 = ((1, 2), 3)
+a5 = (1), (2)
+a6 = ((1), (2))
 
 # Wrapping parentheses checks
 b1 = (

crates/ruff_python_formatter/tests/snapshots/format@statement__for.py.snap

@@ -38,6 +38,16 @@ for (x, y) in (z, w):
 # type comment
 for x in (): # type: int
     ...
+
+# Tuple parentheses for iterable.
+for x in 1, 2, 3:
+    pass
+
+for x in (1, 2, 3):
+    pass
+
+for x in 1, 2, 3,:
+    pass
 ```
 
 ## Output
@@ -76,6 +86,20 @@ for x, y in (z, w):
 # type comment
 for x in ():  # type: int
     ...
+
+# Tuple parentheses for iterable.
+for x in 1, 2, 3:
+    pass
+
+for x in (1, 2, 3):
+    pass
+
+for x in (
+    1,
+    2,
+    3,
+):
+    pass
 ```