Add general support for parenthesized comments on expressions (#6485)

## Summary

This PR adds support for parenthesized comments. A parenthesized comment
is a comment that appears within a parenthesis, but not within the range
of the expression enclosed by the parenthesis. For example, the comment
here is a parenthesized comment:

```python
if (
    # comment
    True
):
    ...
```

The parentheses enclose the `True`, but the range of `True` doesn’t
include the `# comment`.

There are at least two problems associated with parenthesized comments:
(1) associating the comment with the correct (i.e., enclosed) node; and
(2) formatting the comment correctly, once it has been associated with
the enclosed node.

The solution proposed here for (1) is to search for parentheses between
preceding and following node, and use open and close parentheses to
break ties, rather than always assigning to the preceding node.

For (2), we handle these special parenthesized comments in `FormatExpr`.
The biggest risk with this approach is that we forget some codepath that
force-disables parenthesization (by passing in `Parentheses::Never`).
I've audited all usages of that enum and added additional handling +
test coverage for such cases.

Closes https://github.com/astral-sh/ruff/issues/6390.

## Test Plan

`cargo test` with new cases.

Before:

| project      | similarity index |
|--------------|------------------|
| build        | 0.75623          |
| cpython      | 0.75472          |
| django       | 0.99804          |
| transformers | 0.99618          |
| typeshed     | 0.74233          |
| warehouse    | 0.99601          |
| zulip        | 0.99727          |

After:

| project      | similarity index |
|--------------|------------------|
| build        | 0.75623          |
| cpython      | 0.75472          |
| django       | 0.99804          |
| transformers | 0.99618          |
| typeshed     | 0.74237          |
| warehouse    | 0.99601          |
| zulip        | 0.99727          |

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__comments6.py.snap

@@ -141,7 +141,7 @@ aaaaaaaaaaaaa, bbbbbbbbb = map(list, map(itertools.chain.from_iterable, zip(*ite
      an_element_with_a_long_value = calls() or more_calls() and more()  # type: bool
  
      tup = (
-@@ -100,19 +98,30 @@
+@@ -100,7 +98,13 @@
      )
  
      c = call(
@@ -156,10 +156,9 @@ aaaaaaaaaaaaa, bbbbbbbbb = map(list, map(itertools.chain.from_iterable, zip(*ite
      )
  
  
--result = (  # aaa
--    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
--)
-+result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+@@ -108,11 +112,18 @@
+     "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+ )
  
 -AAAAAAAAAAAAA = [AAAAAAAAAAAAA] + SHARED_AAAAAAAAAAAAA + USER_AAAAAAAAAAAAA + AAAAAAAAAAAAA  # type: ignore
 +AAAAAAAAAAAAA = (
@@ -293,7 +292,9 @@ def func(
     )
 
 
-result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+result = (  # aaa
+    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+)
 
 AAAAAAAAAAAAA = (
     [AAAAAAAAAAAAA] + SHARED_AAAAAAAAAAAAA + USER_AAAAAAAAAAAAA + AAAAAAAAAAAAA

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__multiline_consecutive_open_parentheses_ignore.py.snap

@@ -33,15 +33,14 @@ print(   "111"       ) # type: ignore
 ```diff
 --- Black
 +++ Ruff
-@@ -1,9 +1,9 @@
- # This is a regression test. Issue #3737
- 
--a = (  # type: ignore
-+a = int(  # type: ignore  # type: ignore
+@@ -3,7 +3,9 @@
+ a = (  # type: ignore
      int(  # type: ignore
          int(  # type: ignore
 -            int(6)  # type: ignore
-+            6
++            int(  # type: ignore
++                6
++            )
          )
      )
  )
@@ -52,10 +51,12 @@ print(   "111"       ) # type: ignore
 ```py
 # This is a regression test. Issue #3737
 
-a = int(  # type: ignore  # type: ignore
+a = (  # type: ignore
     int(  # type: ignore
         int(  # type: ignore
-            6
+            int(  # type: ignore
+                6
+            )
         )
     )
 )

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__remove_await_parens.py.snap

@@ -93,13 +93,12 @@ async def main():
 ```diff
 --- Black
 +++ Ruff
-@@ -21,7 +21,10 @@
+@@ -21,7 +21,9 @@
  
  # Check comments
  async def main():
 -    await asyncio.sleep(1)  # Hello
-+    await (
-+        # Hello
++    await (  # Hello
 +        asyncio.sleep(1)
 +    )
  
@@ -133,8 +132,7 @@ async def main():
 
 # Check comments
 async def main():
-    await (
-        # Hello
+    await (  # Hello
         asyncio.sleep(1)
     )
 

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__return_annotation_brackets.py.snap

@@ -100,7 +100,16 @@ def foo() -> tuple[int, int, int,]:
 ```diff
 --- Black
 +++ Ruff
-@@ -26,7 +26,11 @@
+@@ -22,11 +22,19 @@
+ 
+ 
+ # Don't lose the comments
+-def double(a: int) -> int:  # Hello
++def double(
++    a: int
++) -> (  # Hello
++    int
++):
      return 2 * a
  
  
@@ -142,7 +151,11 @@ def double(a: int) -> int:
 
 
 # Don't lose the comments
-def double(a: int) -> int:  # Hello
+def double(
+    a: int
+) -> (  # Hello
+    int
+):
     return 2 * a
 
 

crates/ruff_python_formatter/tests/snapshots/format@expression__attribute.py.snap

@@ -204,8 +204,7 @@ x6 = (
 )
 
 # regression: https://github.com/astral-sh/ruff/issues/6181
-(
-    #
+(  #
     ()
 ).a
 ```

crates/ruff_python_formatter/tests/snapshots/format@expression__call.py.snap

@@ -122,11 +122,54 @@ threshold_date = datetime.datetime.now() - datetime.timedelta(  # comment
     days=threshold_days_threshold_days
 )
 
-f(
-    (  # comment
+# Parenthesized and opening-parenthesis comments
+func(
+    (x for x in y)
+)
+
+func(  # outer comment
+    (x for x in y)
+)
+
+func(
+    ( # inner comment
+        x for x in y
+    )
+)
+
+func(
+    (
+        # inner comment
+        x for x in y
+    )
+)
+
+func(  # outer comment
+    (  # inner comment
         1
     )
 )
+
+func(
+    # outer comment
+    ( # inner comment
+        x for x in y
+    )
+)
+
+
+func(
+    ( # inner comment
+        []
+    )
+)
+
+func(
+    # outer comment
+    ( # inner comment
+        []
+    )
+)
 ```
 
 ## Output
@@ -248,12 +291,52 @@ threshold_date = datetime.datetime.now() - datetime.timedelta(  # comment
     days=threshold_days_threshold_days
 )
 
-f(
-    (
-        # comment
+# Parenthesized and opening-parenthesis comments
+func(x for x in y)
+
+func(  # outer comment
+    x for x in y
+)
+
+func(
+    (  # inner comment
+        x for x in y
+    )
+)
+
+func(
+    # inner comment
+    x
+    for x in y
+)
+
+func(  # outer comment
+    (  # inner comment
         1
     )
 )
+
+func(
+    # outer comment
+    (  # inner comment
+        x for x in y
+    )
+)
+
+
+func(
+    (  # inner comment
+        []
+    )
+)
+
+func(
+    (
+        # outer comment
+        # inner comment
+        []
+    )
+)
 ```
 
 

crates/ruff_python_formatter/tests/snapshots/format@expression__generator_exp.py.snap

@@ -79,10 +79,11 @@ f((1) for _ in (a))
 # black keeps these atm, but intends to remove them in the future:
 # https://github.com/psf/black/issues/2943
 len(
-    # leading
-    a
-    for b in c
-    # trailing
+    (  # leading
+        a
+        for b in c
+        # trailing
+    )
 )
 
 len(

crates/ruff_python_formatter/tests/snapshots/format@expression__starred.py.snap

@@ -19,6 +19,16 @@ call(
         [What, i, this, s, very, long, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
     ) # trailing value comment
 )
+
+call(
+    x,
+    # Leading starred comment
+    * # Trailing star comment
+    [
+        # Leading value comment
+        [What, i, this, s, very, long, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
+    ] # trailing value comment
+)
 ```
 
 ## Output
@@ -55,6 +65,24 @@ call(
         ]
     )  # trailing value comment
 )
+
+call(
+    x,
+    # Leading starred comment
+    # Trailing star comment
+    *[
+        # Leading value comment
+        [
+            What,
+            i,
+            this,
+            s,
+            very,
+            long,
+            aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,
+        ]
+    ],  # trailing value comment
+)
 ```
 
 

crates/ruff_python_formatter/tests/snapshots/format@parentheses__nested.py.snap

@@ -89,8 +89,11 @@ a = (
     a
     + b
     + c
-    + d  # Hello
-    + (e + f + g)
+    + d
+    +
+    (  # Hello
+        e + f + g
+    )
 )
 
 a = int(  # type: ignore

crates/ruff_python_formatter/tests/snapshots/format@parentheses__opening_parentheses_comment_value.py.snap

@@ -79,7 +79,49 @@ def e2() -> (  # e 2
 x): pass
 
 
-class E3(  # e 3
+def e3() -> (
+    # e 2
+x): pass
+
+
+def e4() -> (
+    x
+# e 4
+): pass
+
+
+def e5() -> (  # e 5
+    (  # e 5
+        x
+    )
+): pass
+
+
+def e6() -> (
+    (
+        # e 6
+        x
+    )
+): pass
+
+
+def e7() -> (
+    (
+        x
+        # e 7
+    )
+): pass
+
+
+def e8() -> (
+    (
+        x
+    )
+    # e 8
+): pass
+
+
+class E9(  # e 9
 x): pass
 
 
@@ -118,54 +160,54 @@ f5 = {  # f5
 ```py
 # Opening parentheses end-of-line comment with value in the parentheses
 
-(
-    # a 1
+(  # a 1
+    x
+)
+a2 = (  # a 2
     x
 )
-a2 = x  # a 2
 a3 = f(  # a 3
     x
 )
-a4 = x = a4  # a 4
+a4 = (  # a 4
+    x
+) = a4
 a5: List(  # a 5
     x
 ) = 5
 
-raise (
-    # b 1a
+raise (  # b 1a
     x
 )
-raise b1b from (x)  # b 1b
-raise (
-    # b 1c
+raise b1b from (  # b 1b
+    x
+)
+raise (  # b 1c
     x
 ) from b1c
-del (
-    # b 2
+del (  # b 2
+    x
+)
+assert (  # b 3
+    x
+), (  # b 4
     x
 )
-assert (
-    # b 3
-    x  # b 4
-), x
 
 
 def g():
     """Statements that are only allowed in function bodies"""
-    return (
-        # c 1
+    return (  # c 1
         x
     )
-    yield (
-        # c 2
+    yield (  # c 2
         x
     )
 
 
 async def h():
     """Statements that are only allowed in async function bodies"""
-    await (
-        # c 3
+    await (  # c 3
         x
     )
 
@@ -174,8 +216,7 @@ with (  # d 1
     x
 ):
     pass
-match (
-    # d 2
+match (  # d 2
     x
 ):
     case NOT_YET_IMPLEMENTED_Pattern:
@@ -183,30 +224,27 @@ match (
 match d3:
     case NOT_YET_IMPLEMENTED_Pattern:
         pass
-while (
-    # d 4
+while (  # d 4
     x
 ):
     pass
-if (
-    # d 5
+if (  # d 5
     x
 ):
     pass
-elif (
-    # d 6
+elif (  # d 6
     y
 ):
     pass
-for (
-    # d 7
-    x  # d 8
-) in y:
+for (  # d 7
+    x
+) in (  # d 8
+    y
+):
     pass
 try:
     pass
-except (
-    # d 9
+except (  # d 9
     x
 ):
     pass
@@ -218,11 +256,55 @@ def e1(  # e 1
     pass
 
 
-def e2() -> x:  # e 2
+def e2() -> (  # e 2
+    x
+):
     pass
 
 
-class E3(  # e 3
+def e3() -> (
+    # e 2
+    x
+):
+    pass
+
+
+def e4() -> (
+    x
+    # e 4
+):
+    pass
+
+
+def e5() -> (  # e 5
+    # e 5
+    x
+):
+    pass
+
+
+def e6() -> (
+    # e 6
+    x
+):
+    pass
+
+
+def e7() -> (
+    x
+    # e 7
+):
+    pass
+
+
+def e8() -> (
+    x
+    # e 8
+):
+    pass
+
+
+class E9(  # e 9
     x
 ):
     pass

crates/ruff_python_formatter/tests/snapshots/format@statement__assert.py.snap

@@ -52,10 +52,8 @@ assert (  # Dangle1
 assert (
     # Leading test value
     True  # Trailing test value same-line
-), (
     # Trailing test value own-line
-    "Some string"
-)  # Trailing msg same-line
+), "Some string"  # Trailing msg same-line
 # Trailing assert
 
 # Random dangler
@@ -65,11 +63,9 @@ assert (
 assert (
     # Leading test value
     True  # Trailing test value same-line
-), (
     # Trailing test value own-line
     # Test dangler
-    "Some string"
-)  # Trailing msg same-line
+), "Some string"  # Trailing msg same-line
 # Trailing assert
 ```
 

crates/ruff_python_formatter/tests/snapshots/format@statement__assign.py.snap

@@ -33,6 +33,33 @@ aa = ([
 aaaa = ( # trailing
     # comment
     bbbbb) = cccccccccccccccc = 3
+
+x = (  # comment
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    # comment
+    [
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
 ```
 
 ## Output
@@ -70,6 +97,31 @@ aaaa = (  # trailing
     # comment
     bbbbb
 ) = cccccccccccccccc = 3
+
+x = (  # comment
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    # comment
+    [
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = [  # comment
+    a,
+    b,
+    c,
+] = 1
 ```
 
 

crates/ruff_python_formatter/tests/snapshots/format@statement__match.py.snap

@@ -125,8 +125,7 @@ match foo:  # dangling match comment
 
 
 # leading match comment
-match (
-    # leading expr comment
+match (  # leading expr comment
     # another leading expr comment
     foo  # trailing expr comment
     # another trailing expr comment
@@ -151,16 +150,14 @@ match [  # comment
     case NOT_YET_IMPLEMENTED_Pattern:
         pass
 
-match (
-    # comment
+match (  # comment
     "a b c"
 ).split():  # another comment
     case NOT_YET_IMPLEMENTED_Pattern:
         pass
 
 
-match (
-    # comment
+match (  # comment
     # let's go
     yield foo
 ):  # another comment

crates/ruff_python_formatter/tests/snapshots/format@statement__raise.py.snap

@@ -154,8 +154,7 @@ raise (
 )
 
 
-raise (
-    # hey
+raise (  # hey
     aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
     # Holala
     + bbbbbbbbbbbbbbbbbbbbbbbbbb  # stay
@@ -165,8 +164,7 @@ raise (
 )  # whaaaaat
 # the end
 
-raise (
-    # hey 2
+raise (  # hey 2
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
     # Holala
     "bbbbbbbbbbbbbbbbbbbbbbbb"  # stay

crates/ruff_python_formatter/tests/snapshots/format@statement__return_annotation.py.snap

@@ -223,7 +223,9 @@ def zrevrangebylex(
     min: _Value,
     start: int | None = None,
     num: int | None = None,
-) -> 1:  # type: ignore[override]
+) -> (  # type: ignore[override]
+    1
+):
     ...
 
 
@@ -248,7 +250,9 @@ def zrevrangebylex(
     min: _Value,
     start: int | None = None,
     num: int | None = None,
-) -> (1, 2):  # type: ignore[override]
+) -> (  # type: ignore[override]
+    (1, 2)
+):
     ...
 
 
@@ -264,7 +268,11 @@ def double(
     return 2 * a
 
 
-def double(a: int) -> int:  # Hello
+def double(
+    a: int
+) -> (  # Hello
+    int
+):
     return 2 * a
 
 

crates/ruff_python_formatter/tests/snapshots/format@statement__with.py.snap

@@ -114,7 +114,6 @@ with (
 ):
     ...
 
-
 with [
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
     "bbbbbbbbbb",
@@ -131,7 +130,6 @@ with (  # comment
 ):
     ...
 
-
 with (  # outer comment
     (  # inner comment
         CtxManager1()
@@ -140,6 +138,32 @@ with (  # outer comment
     CtxManager3() as example3,
 ):
     ...
+
+with (  # outer comment
+    CtxManager()
+) as example:
+    ...
+
+with (  # outer comment
+    CtxManager()
+) as example, (  # inner comment
+    CtxManager2()
+) as example2:
+    ...
+
+with (  # outer comment
+    CtxManager1(),
+    CtxManager2(),
+) as example:
+    ...
+
+with (  # outer comment
+    (  # inner comment
+        CtxManager1()
+    ),
+    CtxManager2(),
+) as example:
+    ...
 ```
 
 ## Output
@@ -260,7 +284,6 @@ with (
 ):
     ...
 
-
 with [
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
     "bbbbbbbbbb",
@@ -277,16 +300,40 @@ with (  # comment
 ):
     ...
 
-
 with (  # outer comment
-    (
-        # inner comment
+    (  # inner comment
         CtxManager1()
     ) as example1,
     CtxManager2() as example2,
     CtxManager3() as example3,
 ):
     ...
+
+with (  # outer comment
+    CtxManager()
+) as example:
+    ...
+
+with (  # outer comment
+    CtxManager()
+) as example, (  # inner comment
+    CtxManager2()
+) as example2:
+    ...
+
+with (  # outer comment
+    CtxManager1(),
+    CtxManager2(),
+) as example:
+    ...
+
+with (  # outer comment
+    (  # inner comment
+        CtxManager1()
+    ),
+    CtxManager2(),
+) as example:
+    ...
 ```