Add general support for parenthesized comments on expressions (#6485)

## Summary

This PR adds support for parenthesized comments. A parenthesized comment
is a comment that appears within a parenthesis, but not within the range
of the expression enclosed by the parenthesis. For example, the comment
here is a parenthesized comment:

```python
if (
    # comment
    True
):
    ...
```

The parentheses enclose the `True`, but the range of `True` doesn’t
include the `# comment`.

There are at least two problems associated with parenthesized comments:
(1) associating the comment with the correct (i.e., enclosed) node; and
(2) formatting the comment correctly, once it has been associated with
the enclosed node.

The solution proposed here for (1) is to search for parentheses between
preceding and following node, and use open and close parentheses to
break ties, rather than always assigning to the preceding node.

For (2), we handle these special parenthesized comments in `FormatExpr`.
The biggest risk with this approach is that we forget some codepath that
force-disables parenthesization (by passing in `Parentheses::Never`).
I've audited all usages of that enum and added additional handling +
test coverage for such cases.

Closes https://github.com/astral-sh/ruff/issues/6390.

## Test Plan

`cargo test` with new cases.

Before:

| project      | similarity index |
|--------------|------------------|
| build        | 0.75623          |
| cpython      | 0.75472          |
| django       | 0.99804          |
| transformers | 0.99618          |
| typeshed     | 0.74233          |
| warehouse    | 0.99601          |
| zulip        | 0.99727          |

After:

| project      | similarity index |
|--------------|------------------|
| build        | 0.75623          |
| cpython      | 0.75472          |
| django       | 0.99804          |
| transformers | 0.99618          |
| typeshed     | 0.74237          |
| warehouse    | 0.99601          |
| zulip        | 0.99727          |

crates/ruff_python_formatter/resources/test/fixtures/ruff/expression/call.py

@@ -116,8 +116,51 @@ threshold_date = datetime.datetime.now() - datetime.timedelta(  # comment
     days=threshold_days_threshold_days
 )
 
-f(
-    (  # comment
+# Parenthesized and opening-parenthesis comments
+func(
+    (x for x in y)
+)
+
+func(  # outer comment
+    (x for x in y)
+)
+
+func(
+    ( # inner comment
+        x for x in y
+    )
+)
+
+func(
+    (
+        # inner comment
+        x for x in y
+    )
+)
+
+func(  # outer comment
+    (  # inner comment
         1
     )
 )
+
+func(
+    # outer comment
+    ( # inner comment
+        x for x in y
+    )
+)
+
+
+func(
+    ( # inner comment
+        []
+    )
+)
+
+func(
+    # outer comment
+    ( # inner comment
+        []
+    )
+)

crates/ruff_python_formatter/resources/test/fixtures/ruff/expression/starred.py

@@ -13,3 +13,13 @@ call(
         [What, i, this, s, very, long, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
     ) # trailing value comment
 )
+
+call(
+    x,
+    # Leading starred comment
+    * # Trailing star comment
+    [
+        # Leading value comment
+        [What, i, this, s, very, long, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
+    ] # trailing value comment
+)

crates/ruff_python_formatter/resources/test/fixtures/ruff/parentheses/opening_parentheses_comment_value.py

@@ -73,7 +73,49 @@ def e2() -> (  # e 2
 x): pass
 
 
-class E3(  # e 3
+def e3() -> (
+    # e 2
+x): pass
+
+
+def e4() -> (
+    x
+# e 4
+): pass
+
+
+def e5() -> (  # e 5
+    (  # e 5
+        x
+    )
+): pass
+
+
+def e6() -> (
+    (
+        # e 6
+        x
+    )
+): pass
+
+
+def e7() -> (
+    (
+        x
+        # e 7
+    )
+): pass
+
+
+def e8() -> (
+    (
+        x
+    )
+    # e 8
+): pass
+
+
+class E9(  # e 9
 x): pass
 
 

crates/ruff_python_formatter/resources/test/fixtures/ruff/statement/assign.py

@@ -27,3 +27,30 @@ aa = ([
 aaaa = ( # trailing
     # comment
     bbbbb) = cccccccccccccccc = 3
+
+x = (  # comment
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    # comment
+    [
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1

crates/ruff_python_formatter/resources/test/fixtures/ruff/statement/with.py

@@ -108,7 +108,6 @@ with (
 ):
     ...
 
-
 with [
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
     "bbbbbbbbbb",
@@ -125,7 +124,6 @@ with (  # comment
 ):
     ...
 
-
 with (  # outer comment
     (  # inner comment
         CtxManager1()
@@ -134,3 +132,29 @@ with (  # outer comment
     CtxManager3() as example3,
 ):
     ...
+
+with (  # outer comment
+    CtxManager()
+) as example:
+    ...
+
+with (  # outer comment
+    CtxManager()
+) as example, (  # inner comment
+    CtxManager2()
+) as example2:
+    ...
+
+with (  # outer comment
+    CtxManager1(),
+    CtxManager2(),
+) as example:
+    ...
+
+with (  # outer comment
+    (  # inner comment
+        CtxManager1()
+    ),
+    CtxManager2(),
+) as example:
+    ...

crates/ruff_python_formatter/src/comments/mod.rs

@@ -414,6 +414,24 @@ impl<'a> Comments<'a> {
             .leading_dangling_trailing(&NodeRefEqualityKey::from_ref(node.into()))
     }
 
+    /// Returns any comments on the open parenthesis of a `node`.
+    ///
+    /// For example, `# comment` in:
+    /// ```python
+    /// (  # comment
+    ///    foo.bar
+    /// )
+    /// ```
+    #[inline]
+    pub(crate) fn open_parenthesis_comment<T>(&self, node: T) -> Option<&SourceComment>
+    where
+        T: Into<AnyNodeRef<'a>>,
+    {
+        self.leading_comments(node)
+            .first()
+            .filter(|comment| comment.line_position.is_end_of_line())
+    }
+
     #[inline(always)]
     #[cfg(not(debug_assertions))]
     pub(crate) fn assert_formatted_all_comments(&self, _source_code: SourceCode) {}

crates/ruff_python_formatter/src/comments/placement.rs

@@ -18,16 +18,149 @@ pub(super) fn place_comment<'a>(
     comment: DecoratedComment<'a>,
     locator: &Locator,
 ) -> CommentPlacement<'a> {
-    // Handle comments before and after bodies such as the different branches of an if statement.
-    let comment = if comment.line_position().is_own_line() {
-        handle_own_line_comment_around_body(comment, locator)
-    } else {
-        handle_end_of_line_comment_around_body(comment, locator)
+    handle_parenthesized_comment(comment, locator)
+        .or_else(|comment| handle_end_of_line_comment_around_body(comment, locator))
+        .or_else(|comment| handle_own_line_comment_around_body(comment, locator))
+        .or_else(|comment| handle_enclosed_comment(comment, locator))
+}
+
+/// Handle parenthesized comments. A parenthesized comment is a comment that appears within a
+/// parenthesis, but not within the range of the expression enclosed by the parenthesis.
+/// For example, the comment here is a parenthesized comment:
+/// ```python
+/// if (
+///     # comment
+///     True
+/// ):
+///     ...
+/// ```
+/// The parentheses enclose `True`, but the range of `True`doesn't include the `# comment`.
+///
+/// Default handling can get parenthesized comments wrong in a number of ways. For example, the
+/// comment here is marked (by default) as a trailing comment of `x`, when it should be a leading
+/// comment of `y`:
+/// ```python
+/// assert (
+///     x
+/// ), ( # comment
+///     y
+/// )
+/// ```
+///
+/// Similarly, this is marked as a leading comment of `y`, when it should be a trailing comment of
+/// `x`:
+/// ```python
+/// if (
+///     x
+///     # comment
+/// ):
+///    y
+/// ```
+///
+/// As a generalized solution, if a comment has a preceding node and a following node, we search for
+/// opening and closing parentheses between the two nodes. If we find a closing parenthesis between
+/// the preceding node and the comment, then the comment is a trailing comment of the preceding
+/// node. If we find an opening parenthesis between the comment and the following node, then the
+/// comment is a leading comment of the following node.
+fn handle_parenthesized_comment<'a>(
+    comment: DecoratedComment<'a>,
+    locator: &Locator,
+) -> CommentPlacement<'a> {
+    let Some(preceding) = comment.preceding_node() else {
+        return CommentPlacement::Default(comment);
     };
 
-    // Change comment placement depending on the node type. These can be seen as node-specific
-    // fixups.
-    comment.or_else(|comment| match comment.enclosing_node() {
+    let Some(following) = comment.following_node() else {
+        return CommentPlacement::Default(comment);
+    };
+
+    // TODO(charlie): Assert that there are no bogus tokens in these ranges. There are a few cases
+    // where we _can_ hit bogus tokens, but the parentheses need to come before them. For example:
+    // ```python
+    // try:
+    //     some_call()
+    // except (
+    //     UnformattedError
+    //     # trailing comment
+    // ) as err:
+    //     handle_exception()
+    // ```
+    // Here, we lex from the end of `UnformattedError` to the start of `handle_exception()`, which
+    // means we hit an "other" token at `err`. We know the parentheses must precede the `err`, but
+    // this could be fixed by including `as err` in the node range.
+    //
+    // Another example:
+    // ```python
+    // @deco
+    // # comment
+    // def decorated():
+    //     pass
+    // ```
+    // Here, we lex from the end of `deco` to the start of the arguments of `decorated`. We hit an
+    // "other" token at `decorated`, but any parentheses must precede that.
+    //
+    // For now, we _can_ assert, but to do so, we stop lexing when we hit a token that precedes an
+    // identifier.
+    if comment.line_position().is_end_of_line() {
+        let tokenizer = SimpleTokenizer::new(
+            locator.contents(),
+            TextRange::new(preceding.end(), comment.start()),
+        );
+        if tokenizer
+            .skip_trivia()
+            .take_while(|token| {
+                !matches!(
+                    token.kind,
+                    SimpleTokenKind::As | SimpleTokenKind::Def | SimpleTokenKind::Class
+                )
+            })
+            .any(|token| {
+                debug_assert!(
+                    !matches!(token.kind, SimpleTokenKind::Bogus),
+                    "Unexpected token between nodes: `{:?}`",
+                    locator.slice(TextRange::new(preceding.end(), comment.start()),)
+                );
+
+                token.kind() == SimpleTokenKind::LParen
+            })
+        {
+            return CommentPlacement::leading(following, comment);
+        }
+    } else {
+        let tokenizer = SimpleTokenizer::new(
+            locator.contents(),
+            TextRange::new(comment.end(), following.start()),
+        );
+        if tokenizer
+            .skip_trivia()
+            .take_while(|token| {
+                !matches!(
+                    token.kind,
+                    SimpleTokenKind::As | SimpleTokenKind::Def | SimpleTokenKind::Class
+                )
+            })
+            .any(|token| {
+                debug_assert!(
+                    !matches!(token.kind, SimpleTokenKind::Bogus),
+                    "Unexpected token between nodes: `{:?}`",
+                    locator.slice(TextRange::new(comment.end(), following.start()))
+                );
+                token.kind() == SimpleTokenKind::RParen
+            })
+        {
+            return CommentPlacement::trailing(preceding, comment);
+        }
+    }
+
+    CommentPlacement::Default(comment)
+}
+
+/// Handle a comment that is enclosed by a node.
+fn handle_enclosed_comment<'a>(
+    comment: DecoratedComment<'a>,
+    locator: &Locator,
+) -> CommentPlacement<'a> {
+    match comment.enclosing_node() {
         AnyNodeRef::Parameters(arguments) => {
             handle_parameters_separator_comment(comment, arguments, locator)
                 .or_else(|comment| handle_bracketed_end_of_line_comment(comment, locator))
@@ -65,10 +198,7 @@ pub(super) fn place_comment<'a>(
             handle_module_level_own_line_comment_before_class_or_function_comment(comment, locator)
         }
         AnyNodeRef::WithItem(_) => handle_with_item_comment(comment, locator),
-        AnyNodeRef::StmtFunctionDef(function_def) => {
-            handle_leading_function_with_decorators_comment(comment)
-                .or_else(|comment| handle_leading_returns_comment(comment, function_def))
-        }
+        AnyNodeRef::StmtFunctionDef(_) => handle_leading_function_with_decorators_comment(comment),
         AnyNodeRef::StmtClassDef(class_def) => {
             handle_leading_class_with_decorators_comment(comment, class_def)
         }
@@ -90,13 +220,17 @@ pub(super) fn place_comment<'a>(
         | AnyNodeRef::ExprDictComp(_)
         | AnyNodeRef::ExprTuple(_) => handle_bracketed_end_of_line_comment(comment, locator),
         _ => CommentPlacement::Default(comment),
-    })
+    }
 }
 
 fn handle_end_of_line_comment_around_body<'a>(
     comment: DecoratedComment<'a>,
     locator: &Locator,
 ) -> CommentPlacement<'a> {
+    if comment.line_position().is_own_line() {
+        return CommentPlacement::Default(comment);
+    }
+
     // Handle comments before the first statement in a body
     // ```python
     // for x in range(10): # in the main body ...
@@ -245,7 +379,9 @@ fn handle_own_line_comment_around_body<'a>(
     comment: DecoratedComment<'a>,
     locator: &Locator,
 ) -> CommentPlacement<'a> {
-    debug_assert!(comment.line_position().is_own_line());
+    if comment.line_position().is_end_of_line() {
+        return CommentPlacement::Default(comment);
+    }
 
     // If the following is the first child in an alternative body, this must be the last child in
     // the previous one
@@ -274,18 +410,11 @@ fn handle_own_line_comment_around_body<'a>(
     }
 
     // Check if we're between bodies and should attach to the following body.
-    handle_own_line_comment_between_branches(comment, preceding, locator)
-        .or_else(|comment| {
+    handle_own_line_comment_between_branches(comment, preceding, locator).or_else(|comment| {
         // Otherwise, there's no following branch or the indentation is too deep, so attach to the
         // recursively last statement in the preceding body with the matching indentation.
         handle_own_line_comment_after_branch(comment, preceding, locator)
     })
-        .or_else(|comment| {
-            // If the following node is the first in its body, and there's a non-trivia token between the
-            // comment and the following node (like a parenthesis), then it means the comment is trailing
-            // the preceding node, not leading the following one.
-            handle_own_line_comment_in_clause(comment, preceding, locator)
-        })
 }
 
 /// Handles own line comments between two branches of a node.
@@ -385,36 +514,6 @@ fn handle_own_line_comment_between_branches<'a>(
     }
 }
 
-/// Handles own-line comments at the end of a clause, immediately preceding a body:
-/// ```python
-/// if (
-///     True
-///     # This should be a trailing comment of `True` and not a leading comment of `pass`
-/// ):
-///     pass
-/// ```
-fn handle_own_line_comment_in_clause<'a>(
-    comment: DecoratedComment<'a>,
-    preceding: AnyNodeRef<'a>,
-    locator: &Locator,
-) -> CommentPlacement<'a> {
-    if let Some(following) = comment.following_node() {
-        if is_first_statement_in_body(following, comment.enclosing_node())
-            && SimpleTokenizer::new(
-                locator.contents(),
-                TextRange::new(comment.end(), following.start()),
-            )
-            .skip_trivia()
-            .next()
-            .is_some()
-        {
-            return CommentPlacement::trailing(preceding, comment);
-        }
-    }
-
-    CommentPlacement::Default(comment)
-}
-
 /// Determine where to attach an own line comment after a branch depending on its indentation
 fn handle_own_line_comment_after_branch<'a>(
     comment: DecoratedComment<'a>,
@@ -787,40 +886,6 @@ fn handle_leading_function_with_decorators_comment(comment: DecoratedComment) ->
     }
 }
 
-/// Handles end-of-line comments between function parameters and the return type annotation,
-/// attaching them as dangling comments to the function instead of making them trailing
-/// parameter comments.
-///
-/// ```python
-/// def double(a: int) -> ( # Hello
-///     int
-/// ):
-///     return 2*a
-/// ```
-fn handle_leading_returns_comment<'a>(
-    comment: DecoratedComment<'a>,
-    function_def: &'a ast::StmtFunctionDef,
-) -> CommentPlacement<'a> {
-    let parameters = function_def.parameters.as_ref();
-    let Some(returns) = function_def.returns.as_deref() else {
-        return CommentPlacement::Default(comment);
-    };
-
-    let is_preceding_parameters = comment
-        .preceding_node()
-        .is_some_and(|node| node == parameters.into());
-
-    let is_following_returns = comment
-        .following_node()
-        .is_some_and(|node| node == returns.into());
-
-    if comment.line_position().is_end_of_line() && is_preceding_parameters && is_following_returns {
-        CommentPlacement::dangling(comment.enclosing_node(), comment)
-    } else {
-        CommentPlacement::Default(comment)
-    }
-}
-
 /// Handle comments between decorators and the decorated node.
 ///
 /// For example, given:
@@ -1043,14 +1108,6 @@ fn handle_trailing_expression_starred_star_end_of_line_comment<'a>(
     comment: DecoratedComment<'a>,
     starred: &'a ast::ExprStarred,
 ) -> CommentPlacement<'a> {
-    if comment.line_position().is_own_line() {
-        return CommentPlacement::Default(comment);
-    }
-
-    if comment.following_node().is_none() {
-        return CommentPlacement::Default(comment);
-    }
-
     CommentPlacement::leading(starred, comment)
 }
 

crates/ruff_python_formatter/src/comments/snapshots/ruff_python_formatter__comments__tests__parenthesized_expression.snap

@@ -4,19 +4,19 @@ expression: comments.debug(test_case.source_code)
 ---
 {
     Node {
-        kind: ExprName,
-        range: 1..2,
-        source: `a`,
+        kind: ExprBinOp,
+        range: 30..57,
+        source: `10 + # More comments⏎`,
     }: {
-        "leading": [],
-        "dangling": [],
-        "trailing": [
+        "leading": [
             SourceComment {
                 text: "# Trailing comment",
                 position: EndOfLine,
                 formatted: false,
             },
         ],
+        "dangling": [],
+        "trailing": [],
     },
     Node {
         kind: ExprConstant,

crates/ruff_python_formatter/src/expression/expr_generator_exp.rs

@@ -2,7 +2,7 @@ use ruff_formatter::{format_args, write, Buffer, FormatResult, FormatRuleWithOpt
 use ruff_python_ast::node::AnyNodeRef;
 use ruff_python_ast::ExprGeneratorExp;
 
-use crate::comments::{leading_comments, SourceComment};
+use crate::comments::SourceComment;
 use crate::context::PyFormatContext;
 use crate::expression::parentheses::{parenthesized, NeedsParentheses, OptionalParentheses};
 use crate::prelude::*;
@@ -14,10 +14,11 @@ pub enum GeneratorExpParentheses {
     #[default]
     Default,
 
-    // skip parens if the generator exp is the only argument to a function, e.g.
-    // ```python
-    //  all(x for y in z)`
-    //  ```
+    /// Skip parens if the generator is the only argument to a function and doesn't contain any
+    /// dangling comments. For example:
+    /// ```python
+    /// all(x for y in z)`
+    /// ```
     StripIfOnlyFunctionArg,
 }
 
@@ -52,15 +53,12 @@ impl FormatNodeRule<ExprGeneratorExp> for FormatExprGeneratorExp {
         let comments = f.context().comments().clone();
         let dangling = comments.dangling_comments(item);
 
-        if self.parentheses == GeneratorExpParentheses::StripIfOnlyFunctionArg {
+        if self.parentheses == GeneratorExpParentheses::StripIfOnlyFunctionArg
+            && dangling.is_empty()
+        {
             write!(
                 f,
-                [
-                    leading_comments(dangling),
-                    group(&elt.format()),
-                    soft_line_break_or_space(),
-                    &joined
-                ]
+                [group(&elt.format()), soft_line_break_or_space(), &joined]
             )
         } else {
             write!(

crates/ruff_python_formatter/src/expression/mod.rs

@@ -107,7 +107,15 @@ impl FormatRule<Expr, PyFormatContext<'_>> for FormatExpr {
         };
 
         if parenthesize {
-            parenthesized("(", &format_expr, ")").fmt(f)
+            let comments = f.context().comments().clone();
+            let open_parenthesis_comment = comments.open_parenthesis_comment(expression);
+            parenthesized("(", &format_expr, ")")
+                .with_dangling_comments(
+                    open_parenthesis_comment
+                        .map(std::slice::from_ref)
+                        .unwrap_or_default(),
+                )
+                .fmt(f)
         } else {
             let level = match f.context().node_level() {
                 NodeLevel::TopLevel | NodeLevel::CompoundStatement => NodeLevel::Expression(None),
@@ -162,6 +170,9 @@ impl Format<PyFormatContext<'_>> for MaybeParenthesizeExpression<'_> {
         let has_comments = comments.has_leading_comments(*expression)
             || comments.has_trailing_own_line_comments(*expression);
 
+        // If the expression has comments, we always want to preserve the parentheses. This also
+        // ensures that we correctly handle parenthesized comments, and don't need to worry about
+        // them in the implementation below.
         if preserve_parentheses || has_comments {
             return expression.format().with_options(Parentheses::Always).fmt(f);
         }

crates/ruff_python_formatter/src/other/arguments.rs

@@ -45,6 +45,9 @@ impl FormatNodeRule<Arguments> for FormatArguments {
                                 if is_single_argument_parenthesized(arg, item.end(), source) {
                                     Parentheses::Always
                                 } else {
+                                    // Note: no need to handle opening-parenthesis comments, since
+                                    // an opening-parenthesis comment implies that the argument is
+                                    // parenthesized.
                                     Parentheses::Never
                                 };
                             joiner.entry(other, &other.format().with_options(parentheses))

crates/ruff_python_formatter/src/statement/stmt_assign.rs

@@ -44,6 +44,7 @@ impl FormatNodeRule<StmtAssign> for FormatStmtAssign {
     }
 }
 
+#[derive(Debug)]
 struct FormatTargets<'a> {
     targets: &'a [Expr],
 }
@@ -51,9 +52,17 @@ struct FormatTargets<'a> {
 impl Format<PyFormatContext<'_>> for FormatTargets<'_> {
     fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
         if let Some((first, rest)) = self.targets.split_first() {
-            let can_omit_parentheses = has_own_parentheses(first, f.context()).is_some();
+            let comments = f.context().comments();
 
-            let group_id = if can_omit_parentheses {
+            let parenthesize = if comments.has_leading_comments(first) {
+                ParenthesizeTarget::Always
+            } else if has_own_parentheses(first, f.context()).is_some() {
+                ParenthesizeTarget::Never
+            } else {
+                ParenthesizeTarget::IfBreaks
+            };
+
+            let group_id = if parenthesize == ParenthesizeTarget::Never {
                 Some(f.group_id("assignment_parentheses"))
             } else {
                 None
@@ -61,9 +70,14 @@ impl Format<PyFormatContext<'_>> for FormatTargets<'_> {
 
             let format_first = format_with(|f: &mut PyFormatter| {
                 let mut f = WithNodeLevel::new(NodeLevel::Expression(group_id), f);
-                if can_omit_parentheses {
+                match parenthesize {
+                    ParenthesizeTarget::Always => {
+                        write!(f, [first.format().with_options(Parentheses::Always)])
+                    }
+                    ParenthesizeTarget::Never => {
                         write!(f, [first.format().with_options(Parentheses::Never)])
-                } else {
+                    }
+                    ParenthesizeTarget::IfBreaks => {
                         write!(
                             f,
                             [
@@ -73,6 +87,7 @@ impl Format<PyFormatContext<'_>> for FormatTargets<'_> {
                             ]
                         )
                     }
+                }
             });
 
             write!(
@@ -91,3 +106,10 @@ impl Format<PyFormatContext<'_>> for FormatTargets<'_> {
         }
     }
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ParenthesizeTarget {
+    Always,
+    Never,
+    IfBreaks,
+}

crates/ruff_python_formatter/src/statement/stmt_function_def.rs

@@ -66,10 +66,12 @@ impl FormatNodeRule<StmtFunctionDef> for FormatStmtFunctionDef {
                 write!(f, [space(), text("->"), space()])?;
 
                 if return_annotation.is_tuple_expr() {
-                    write!(
-                        f,
-                        [return_annotation.format().with_options(Parentheses::Never)]
-                    )?;
+                    let parentheses = if comments.has_leading_comments(return_annotation.as_ref()) {
+                        Parentheses::Always
+                    } else {
+                        Parentheses::Never
+                    };
+                    write!(f, [return_annotation.format().with_options(parentheses)])?;
                 } else if comments.has_trailing_comments(return_annotation.as_ref()) {
                     // Intentionally parenthesize any return annotations with trailing comments.
                     // This avoids an instability in cases like:

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__comments6.py.snap

@@ -141,7 +141,7 @@ aaaaaaaaaaaaa, bbbbbbbbb = map(list, map(itertools.chain.from_iterable, zip(*ite
      an_element_with_a_long_value = calls() or more_calls() and more()  # type: bool
  
      tup = (
-@@ -100,19 +98,30 @@
+@@ -100,7 +98,13 @@
      )
  
      c = call(
@@ -156,10 +156,9 @@ aaaaaaaaaaaaa, bbbbbbbbb = map(list, map(itertools.chain.from_iterable, zip(*ite
      )
  
  
--result = (  # aaa
--    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
--)
-+result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+@@ -108,11 +112,18 @@
+     "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+ )
  
 -AAAAAAAAAAAAA = [AAAAAAAAAAAAA] + SHARED_AAAAAAAAAAAAA + USER_AAAAAAAAAAAAA + AAAAAAAAAAAAA  # type: ignore
 +AAAAAAAAAAAAA = (
@@ -293,7 +292,9 @@ def func(
     )
 
 
-result = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"  # aaa
+result = (  # aaa
+    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+)
 
 AAAAAAAAAAAAA = (
     [AAAAAAAAAAAAA] + SHARED_AAAAAAAAAAAAA + USER_AAAAAAAAAAAAA + AAAAAAAAAAAAA

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__multiline_consecutive_open_parentheses_ignore.py.snap

@@ -33,15 +33,14 @@ print(   "111"       ) # type: ignore
 ```diff
 --- Black
 +++ Ruff
-@@ -1,9 +1,9 @@
- # This is a regression test. Issue #3737
- 
--a = (  # type: ignore
-+a = int(  # type: ignore  # type: ignore
+@@ -3,7 +3,9 @@
+ a = (  # type: ignore
      int(  # type: ignore
          int(  # type: ignore
 -            int(6)  # type: ignore
++            int(  # type: ignore
 +                6
++            )
          )
      )
  )
@@ -52,12 +51,14 @@ print(   "111"       ) # type: ignore
 ```py
 # This is a regression test. Issue #3737
 
-a = int(  # type: ignore  # type: ignore
+a = (  # type: ignore
+    int(  # type: ignore
         int(  # type: ignore
             int(  # type: ignore
                 6
             )
         )
+    )
 )
 
 b = int(6)

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__remove_await_parens.py.snap

@@ -93,13 +93,12 @@ async def main():
 ```diff
 --- Black
 +++ Ruff
-@@ -21,7 +21,10 @@
+@@ -21,7 +21,9 @@
  
  # Check comments
  async def main():
 -    await asyncio.sleep(1)  # Hello
-+    await (
-+        # Hello
++    await (  # Hello
 +        asyncio.sleep(1)
 +    )
  
@@ -133,8 +132,7 @@ async def main():
 
 # Check comments
 async def main():
-    await (
-        # Hello
+    await (  # Hello
         asyncio.sleep(1)
     )
 

crates/ruff_python_formatter/tests/snapshots/black_compatibility@simple_cases__return_annotation_brackets.py.snap

@@ -100,7 +100,16 @@ def foo() -> tuple[int, int, int,]:
 ```diff
 --- Black
 +++ Ruff
-@@ -26,7 +26,11 @@
+@@ -22,11 +22,19 @@
+ 
+ 
+ # Don't lose the comments
+-def double(a: int) -> int:  # Hello
++def double(
++    a: int
++) -> (  # Hello
++    int
++):
      return 2 * a
  
  
@@ -142,7 +151,11 @@ def double(a: int) -> int:
 
 
 # Don't lose the comments
-def double(a: int) -> int:  # Hello
+def double(
+    a: int
+) -> (  # Hello
+    int
+):
     return 2 * a
 
 

crates/ruff_python_formatter/tests/snapshots/format@expression__attribute.py.snap

@@ -204,8 +204,7 @@ x6 = (
 )
 
 # regression: https://github.com/astral-sh/ruff/issues/6181
-(
-    #
+(  #
     ()
 ).a
 ```

crates/ruff_python_formatter/tests/snapshots/format@expression__call.py.snap

@@ -122,11 +122,54 @@ threshold_date = datetime.datetime.now() - datetime.timedelta(  # comment
     days=threshold_days_threshold_days
 )
 
-f(
-    (  # comment
+# Parenthesized and opening-parenthesis comments
+func(
+    (x for x in y)
+)
+
+func(  # outer comment
+    (x for x in y)
+)
+
+func(
+    ( # inner comment
+        x for x in y
+    )
+)
+
+func(
+    (
+        # inner comment
+        x for x in y
+    )
+)
+
+func(  # outer comment
+    (  # inner comment
         1
     )
 )
+
+func(
+    # outer comment
+    ( # inner comment
+        x for x in y
+    )
+)
+
+
+func(
+    ( # inner comment
+        []
+    )
+)
+
+func(
+    # outer comment
+    ( # inner comment
+        []
+    )
+)
 ```
 
 ## Output
@@ -248,12 +291,52 @@ threshold_date = datetime.datetime.now() - datetime.timedelta(  # comment
     days=threshold_days_threshold_days
 )
 
-f(
-    (
-        # comment
+# Parenthesized and opening-parenthesis comments
+func(x for x in y)
+
+func(  # outer comment
+    x for x in y
+)
+
+func(
+    (  # inner comment
+        x for x in y
+    )
+)
+
+func(
+    # inner comment
+    x
+    for x in y
+)
+
+func(  # outer comment
+    (  # inner comment
         1
     )
 )
+
+func(
+    # outer comment
+    (  # inner comment
+        x for x in y
+    )
+)
+
+
+func(
+    (  # inner comment
+        []
+    )
+)
+
+func(
+    (
+        # outer comment
+        # inner comment
+        []
+    )
+)
 ```
 
 

crates/ruff_python_formatter/tests/snapshots/format@expression__generator_exp.py.snap

@@ -79,10 +79,11 @@ f((1) for _ in (a))
 # black keeps these atm, but intends to remove them in the future:
 # https://github.com/psf/black/issues/2943
 len(
-    # leading
+    (  # leading
         a
         for b in c
         # trailing
+    )
 )
 
 len(

crates/ruff_python_formatter/tests/snapshots/format@expression__starred.py.snap

@@ -19,6 +19,16 @@ call(
         [What, i, this, s, very, long, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
     ) # trailing value comment
 )
+
+call(
+    x,
+    # Leading starred comment
+    * # Trailing star comment
+    [
+        # Leading value comment
+        [What, i, this, s, very, long, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]
+    ] # trailing value comment
+)
 ```
 
 ## Output
@@ -55,6 +65,24 @@ call(
         ]
     )  # trailing value comment
 )
+
+call(
+    x,
+    # Leading starred comment
+    # Trailing star comment
+    *[
+        # Leading value comment
+        [
+            What,
+            i,
+            this,
+            s,
+            very,
+            long,
+            aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,
+        ]
+    ],  # trailing value comment
+)
 ```
 
 

crates/ruff_python_formatter/tests/snapshots/format@parentheses__nested.py.snap

@@ -89,8 +89,11 @@ a = (
     a
     + b
     + c
-    + d  # Hello
-    + (e + f + g)
+    + d
+    +
+    (  # Hello
+        e + f + g
+    )
 )
 
 a = int(  # type: ignore

crates/ruff_python_formatter/tests/snapshots/format@parentheses__opening_parentheses_comment_value.py.snap

@@ -79,7 +79,49 @@ def e2() -> (  # e 2
 x): pass
 
 
-class E3(  # e 3
+def e3() -> (
+    # e 2
+x): pass
+
+
+def e4() -> (
+    x
+# e 4
+): pass
+
+
+def e5() -> (  # e 5
+    (  # e 5
+        x
+    )
+): pass
+
+
+def e6() -> (
+    (
+        # e 6
+        x
+    )
+): pass
+
+
+def e7() -> (
+    (
+        x
+        # e 7
+    )
+): pass
+
+
+def e8() -> (
+    (
+        x
+    )
+    # e 8
+): pass
+
+
+class E9(  # e 9
 x): pass
 
 
@@ -118,54 +160,54 @@ f5 = {  # f5
 ```py
 # Opening parentheses end-of-line comment with value in the parentheses
 
-(
-    # a 1
+(  # a 1
+    x
+)
+a2 = (  # a 2
     x
 )
-a2 = x  # a 2
 a3 = f(  # a 3
     x
 )
-a4 = x = a4  # a 4
+a4 = (  # a 4
+    x
+) = a4
 a5: List(  # a 5
     x
 ) = 5
 
-raise (
-    # b 1a
+raise (  # b 1a
     x
 )
-raise b1b from (x)  # b 1b
-raise (
-    # b 1c
+raise b1b from (  # b 1b
+    x
+)
+raise (  # b 1c
     x
 ) from b1c
-del (
-    # b 2
+del (  # b 2
+    x
+)
+assert (  # b 3
+    x
+), (  # b 4
     x
 )
-assert (
-    # b 3
-    x  # b 4
-), x
 
 
 def g():
     """Statements that are only allowed in function bodies"""
-    return (
-        # c 1
+    return (  # c 1
         x
     )
-    yield (
-        # c 2
+    yield (  # c 2
         x
     )
 
 
 async def h():
     """Statements that are only allowed in async function bodies"""
-    await (
-        # c 3
+    await (  # c 3
         x
     )
 
@@ -174,8 +216,7 @@ with (  # d 1
     x
 ):
     pass
-match (
-    # d 2
+match (  # d 2
     x
 ):
     case NOT_YET_IMPLEMENTED_Pattern:
@@ -183,30 +224,27 @@ match (
 match d3:
     case NOT_YET_IMPLEMENTED_Pattern:
         pass
-while (
-    # d 4
+while (  # d 4
     x
 ):
     pass
-if (
-    # d 5
+if (  # d 5
     x
 ):
     pass
-elif (
-    # d 6
+elif (  # d 6
     y
 ):
     pass
-for (
-    # d 7
-    x  # d 8
-) in y:
+for (  # d 7
+    x
+) in (  # d 8
+    y
+):
     pass
 try:
     pass
-except (
-    # d 9
+except (  # d 9
     x
 ):
     pass
@@ -218,11 +256,55 @@ def e1(  # e 1
     pass
 
 
-def e2() -> x:  # e 2
+def e2() -> (  # e 2
+    x
+):
     pass
 
 
-class E3(  # e 3
+def e3() -> (
+    # e 2
+    x
+):
+    pass
+
+
+def e4() -> (
+    x
+    # e 4
+):
+    pass
+
+
+def e5() -> (  # e 5
+    # e 5
+    x
+):
+    pass
+
+
+def e6() -> (
+    # e 6
+    x
+):
+    pass
+
+
+def e7() -> (
+    x
+    # e 7
+):
+    pass
+
+
+def e8() -> (
+    x
+    # e 8
+):
+    pass
+
+
+class E9(  # e 9
     x
 ):
     pass

crates/ruff_python_formatter/tests/snapshots/format@statement__assert.py.snap

@@ -52,10 +52,8 @@ assert (  # Dangle1
 assert (
     # Leading test value
     True  # Trailing test value same-line
-), (
     # Trailing test value own-line
-    "Some string"
-)  # Trailing msg same-line
+), "Some string"  # Trailing msg same-line
 # Trailing assert
 
 # Random dangler
@@ -65,11 +63,9 @@ assert (
 assert (
     # Leading test value
     True  # Trailing test value same-line
-), (
     # Trailing test value own-line
     # Test dangler
-    "Some string"
-)  # Trailing msg same-line
+), "Some string"  # Trailing msg same-line
 # Trailing assert
 ```
 

crates/ruff_python_formatter/tests/snapshots/format@statement__assign.py.snap

@@ -33,6 +33,33 @@ aa = ([
 aaaa = ( # trailing
     # comment
     bbbbb) = cccccccccccccccc = 3
+
+x = (  # comment
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    # comment
+    [
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
 ```
 
 ## Output
@@ -70,6 +97,31 @@ aaaa = (  # trailing
     # comment
     bbbbb
 ) = cccccccccccccccc = 3
+
+x = (  # comment
+    [  # comment
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = (
+    # comment
+    [
+        a,
+        b,
+        c,
+    ]
+) = 1
+
+
+x = [  # comment
+    a,
+    b,
+    c,
+] = 1
 ```
 
 

crates/ruff_python_formatter/tests/snapshots/format@statement__match.py.snap

@@ -125,8 +125,7 @@ match foo:  # dangling match comment
 
 
 # leading match comment
-match (
-    # leading expr comment
+match (  # leading expr comment
     # another leading expr comment
     foo  # trailing expr comment
     # another trailing expr comment
@@ -151,16 +150,14 @@ match [  # comment
     case NOT_YET_IMPLEMENTED_Pattern:
         pass
 
-match (
-    # comment
+match (  # comment
     "a b c"
 ).split():  # another comment
     case NOT_YET_IMPLEMENTED_Pattern:
         pass
 
 
-match (
-    # comment
+match (  # comment
     # let's go
     yield foo
 ):  # another comment

crates/ruff_python_formatter/tests/snapshots/format@statement__raise.py.snap

@@ -154,8 +154,7 @@ raise (
 )
 
 
-raise (
-    # hey
+raise (  # hey
     aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
     # Holala
     + bbbbbbbbbbbbbbbbbbbbbbbbbb  # stay
@@ -165,8 +164,7 @@ raise (
 )  # whaaaaat
 # the end
 
-raise (
-    # hey 2
+raise (  # hey 2
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
     # Holala
     "bbbbbbbbbbbbbbbbbbbbbbbb"  # stay

crates/ruff_python_formatter/tests/snapshots/format@statement__return_annotation.py.snap

@@ -223,7 +223,9 @@ def zrevrangebylex(
     min: _Value,
     start: int | None = None,
     num: int | None = None,
-) -> 1:  # type: ignore[override]
+) -> (  # type: ignore[override]
+    1
+):
     ...
 
 
@@ -248,7 +250,9 @@ def zrevrangebylex(
     min: _Value,
     start: int | None = None,
     num: int | None = None,
-) -> (1, 2):  # type: ignore[override]
+) -> (  # type: ignore[override]
+    (1, 2)
+):
     ...
 
 
@@ -264,7 +268,11 @@ def double(
     return 2 * a
 
 
-def double(a: int) -> int:  # Hello
+def double(
+    a: int
+) -> (  # Hello
+    int
+):
     return 2 * a
 
 

crates/ruff_python_formatter/tests/snapshots/format@statement__with.py.snap

@@ -114,7 +114,6 @@ with (
 ):
     ...
 
-
 with [
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
     "bbbbbbbbbb",
@@ -131,7 +130,6 @@ with (  # comment
 ):
     ...
 
-
 with (  # outer comment
     (  # inner comment
         CtxManager1()
@@ -140,6 +138,32 @@ with (  # outer comment
     CtxManager3() as example3,
 ):
     ...
+
+with (  # outer comment
+    CtxManager()
+) as example:
+    ...
+
+with (  # outer comment
+    CtxManager()
+) as example, (  # inner comment
+    CtxManager2()
+) as example2:
+    ...
+
+with (  # outer comment
+    CtxManager1(),
+    CtxManager2(),
+) as example:
+    ...
+
+with (  # outer comment
+    (  # inner comment
+        CtxManager1()
+    ),
+    CtxManager2(),
+) as example:
+    ...
 ```
 
 ## Output
@@ -260,7 +284,6 @@ with (
 ):
     ...
 
-
 with [
     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
     "bbbbbbbbbb",
@@ -277,16 +300,40 @@ with (  # comment
 ):
     ...
 
-
 with (  # outer comment
-    (
-        # inner comment
+    (  # inner comment
         CtxManager1()
     ) as example1,
     CtxManager2() as example2,
     CtxManager3() as example3,
 ):
     ...
+
+with (  # outer comment
+    CtxManager()
+) as example:
+    ...
+
+with (  # outer comment
+    CtxManager()
+) as example, (  # inner comment
+    CtxManager2()
+) as example2:
+    ...
+
+with (  # outer comment
+    CtxManager1(),
+    CtxManager2(),
+) as example:
+    ...
+
+with (  # outer comment
+    (  # inner comment
+        CtxManager1()
+    ),
+    CtxManager2(),
+) as example:
+    ...
 ```
 
 

crates/ruff_python_trivia/src/tokenizer.rs

@@ -168,6 +168,9 @@ pub enum SimpleTokenKind {
     /// `:`
     Colon,
 
+    /// `;`
+    Semi,
+
     /// '/'
     Slash,
 
@@ -200,6 +203,7 @@ pub enum SimpleTokenKind {
 
     /// `^`
     Circumflex,
+
     /// `|`
     Vbar,
 
@@ -226,6 +230,7 @@ pub enum SimpleTokenKind {
 
     /// `break`
     Break,
+
     /// `class`
     Class,
 
@@ -331,6 +336,7 @@ impl SimpleTokenKind {
             '}' => SimpleTokenKind::RBrace,
             ',' => SimpleTokenKind::Comma,
             ':' => SimpleTokenKind::Colon,
+            ';' => SimpleTokenKind::Semi,
             '/' => SimpleTokenKind::Slash,
             '*' => SimpleTokenKind::Star,
             '.' => SimpleTokenKind::Dot,