mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-04 02:38:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Use CommentRanges in backwards lexing (#7360)

Browse source 
## Summary

The tokenizer was split into a forward and a backwards tokenizer. The
backwards tokenizer uses the same names as the forwards ones (e.g.
`next_token`). The backwards tokenizer gets the comment ranges that we
already built to skip comments.

---------

Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
konsti 2023-09-16 05:21:45 +02:00 • committed by GitHub
parent 1f6e1485f9
commit 2cbe1733c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 744 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/ruff_python_formatter/src/comments/debug.rs
View file

@ -182,6 +182,7 @@ mod tests {
use ruff_formatter::SourceCode;
use ruff_python_ast::node::AnyNode;
use ruff_python_ast::{StmtBreak, StmtContinue};
use ruff_python_trivia::CommentRanges;
use ruff_text_size::{TextRange, TextSize};
use crate::comments::map::MultiMap;
@ -231,7 +232,8 @@ break;
),
);
let comments = Comments::new(comments_map);
let comment_ranges = CommentRanges::default();
let comments = Comments::new(comments_map, &comment_ranges);
assert_debug_snapshot!(comments.debug(source_code));
}

31
crates/ruff_python_formatter/src/comments/mod.rs
View file

@ -99,8 +99,7 @@ use ruff_formatter::{SourceCode, SourceCodeSlice};
use ruff_python_ast::node::AnyNodeRef;
use ruff_python_ast::visitor::preorder::{PreorderVisitor, TraversalSignal};
use ruff_python_ast::Mod;
use ruff_python_index::CommentRanges;
use ruff_python_trivia::PythonWhitespace;
use ruff_python_trivia::{CommentRanges, PythonWhitespace};
use ruff_source_file::Locator;
use ruff_text_size::{Ranged, TextRange};
@ -281,7 +280,7 @@ type CommentsMap<'a> = MultiMap<NodeRefEqualityKey<'a>, SourceComment>;
/// The comments of a syntax tree stored by node.
///
/// Cloning `comments` is cheap as it only involves bumping a reference counter.
#[derive(Debug, Clone, Default)]
#[derive(Debug, Clone)]
pub(crate) struct Comments<'a> {
/// The implementation uses an [Rc] so that [Comments] has a lifetime independent from the [crate::Formatter].
/// Independent lifetimes are necessary to support the use case where a (formattable object)[crate::Format]
@ -306,15 +305,31 @@ pub(crate) struct Comments<'a> {
/// }
/// ```
data: Rc<CommentsData<'a>>,
/// We need those for backwards lexing
comment_ranges: &'a CommentRanges,
}
impl<'a> Comments<'a> {
fn new(comments: CommentsMap<'a>) -> Self {
fn new(comments: CommentsMap<'a>, comment_ranges: &'a CommentRanges) -> Self {
Self {
data: Rc::new(CommentsData { comments }),
comment_ranges,
}
}
/// Effectively a [`Default`] implementation that works around the lifetimes for tests
#[cfg(test)]
pub(crate) fn from_ranges(comment_ranges: &'a CommentRanges) -> Self {
Self {
data: Rc::new(CommentsData::default()),
comment_ranges,
}
}
pub(crate) fn ranges(&self) -> &'a CommentRanges {
self.comment_ranges
}
/// Extracts the comments from the AST.
pub(crate) fn from_ast(
root: &'a Mod,
@ -324,12 +339,13 @@ impl<'a> Comments<'a> {
let map = if comment_ranges.is_empty() {
CommentsMap::new()
} else {
let mut builder = CommentsMapBuilder::new(Locator::new(source_code.as_str()));
let mut builder =
CommentsMapBuilder::new(Locator::new(source_code.as_str()), comment_ranges);
CommentsVisitor::new(source_code, comment_ranges, &mut builder).visit(root);
builder.finish()
};
Self::new(map)
Self::new(map, comment_ranges)
}
/// Returns `true` if the given `node` has any comments.
@ -528,9 +544,10 @@ mod tests {
use ruff_formatter::SourceCode;
use ruff_python_ast::Mod;
use ruff_python_index::{CommentRanges, CommentRangesBuilder};
use ruff_python_index::CommentRangesBuilder;
use ruff_python_parser::lexer::lex;
use ruff_python_parser::{parse_tokens, Mode};
use ruff_python_trivia::CommentRanges;
use crate::comments::Comments;

18
crates/ruff_python_formatter/src/comments/placement.rs
View file

@ -4,7 +4,8 @@ use ruff_python_ast::node::AnyNodeRef;
use ruff_python_ast::whitespace::indentation;
use ruff_python_ast::{self as ast, Comprehension, Expr, MatchCase, Parameters};
use ruff_python_trivia::{
find_only_token_in_range, indentation_at_offset, SimpleToken, SimpleTokenKind, SimpleTokenizer,
find_only_token_in_range, indentation_at_offset, BackwardsTokenizer, CommentRanges,
SimpleToken, SimpleTokenKind, SimpleTokenizer,
};
use ruff_source_file::Locator;
use ruff_text_size::{Ranged, TextLen, TextRange};
@ -20,12 +21,13 @@ use crate::pattern::pattern_match_sequence::SequenceType;
/// Manually attach comments to nodes that the default placement gets wrong.
pub(super) fn place_comment<'a>(
comment: DecoratedComment<'a>,
comment_ranges: &CommentRanges,
locator: &Locator,
) -> CommentPlacement<'a> {
handle_parenthesized_comment(comment, locator)
.or_else(|comment| handle_end_of_line_comment_around_body(comment, locator))
.or_else(|comment| handle_own_line_comment_around_body(comment, locator))
.or_else(|comment| handle_enclosed_comment(comment, locator))
.or_else(|comment| handle_enclosed_comment(comment, comment_ranges, locator))
}
/// Handle parenthesized comments. A parenthesized comment is a comment that appears within a
@ -172,6 +174,7 @@ fn handle_parenthesized_comment<'a>(
/// Handle a comment that is enclosed by a node.
fn handle_enclosed_comment<'a>(
comment: DecoratedComment<'a>,
comment_ranges: &CommentRanges,
locator: &Locator,
) -> CommentPlacement<'a> {
match comment.enclosing_node() {
@ -213,13 +216,15 @@ fn handle_enclosed_comment<'a>(
AnyNodeRef::ExprDict(_) => handle_dict_unpacking_comment(comment, locator)
.or_else(|comment| handle_bracketed_end_of_line_comment(comment, locator)),
AnyNodeRef::ExprIfExp(expr_if) => handle_expr_if_comment(comment, expr_if, locator),
AnyNodeRef::ExprSlice(expr_slice) => handle_slice_comments(comment, expr_slice, locator),
AnyNodeRef::ExprSlice(expr_slice) => {
handle_slice_comments(comment, expr_slice, comment_ranges, locator)
}
AnyNodeRef::ExprStarred(starred) => {
handle_trailing_expression_starred_star_end_of_line_comment(comment, starred, locator)
}
AnyNodeRef::ExprSubscript(expr_subscript) => {
if let Expr::Slice(expr_slice) = expr_subscript.slice.as_ref() {
handle_slice_comments(comment, expr_slice, locator)
handle_slice_comments(comment, expr_slice, comment_ranges, locator)
} else {
CommentPlacement::Default(comment)
}
@ -958,6 +963,7 @@ fn handle_module_level_own_line_comment_before_class_or_function_comment<'a>(
fn handle_slice_comments<'a>(
comment: DecoratedComment<'a>,
expr_slice: &'a ast::ExprSlice,
comment_ranges: &CommentRanges,
locator: &Locator,
) -> CommentPlacement<'a> {
let ast::ExprSlice {
@ -969,9 +975,9 @@ fn handle_slice_comments<'a>(
// Check for `foo[ # comment`, but only if they are on the same line
let after_lbracket = matches!(
SimpleTokenizer::up_to_without_back_comment(comment.start(), locator.contents())
BackwardsTokenizer::up_to(comment.start(), locator.contents(), comment_ranges)
.skip_trivia()
.next_back(),
.next(),
Some(SimpleToken {
kind: SimpleTokenKind::LBracket,
..

10
crates/ruff_python_formatter/src/comments/visitor.rs
View file

@ -8,8 +8,7 @@ use ruff_python_ast::{Mod, Stmt};
// pre-order.
#[allow(clippy::wildcard_imports)]
use ruff_python_ast::visitor::preorder::*;
use ruff_python_index::CommentRanges;
use ruff_python_trivia::is_python_whitespace;
use ruff_python_trivia::{is_python_whitespace, CommentRanges};
use ruff_source_file::Locator;
use ruff_text_size::{Ranged, TextRange, TextSize};
@ -536,12 +535,14 @@ impl<'a> PushComment<'a> for CommentsVecBuilder<'a> {
/// [`CommentsMap`].
pub(super) struct CommentsMapBuilder<'a> {
comments: CommentsMap<'a>,
/// We need those for backwards lexing
comment_ranges: &'a CommentRanges,
locator: Locator<'a>,
}
impl<'a> PushComment<'a> for CommentsMapBuilder<'a> {
fn push_comment(&mut self, placement: DecoratedComment<'a>) {
let placement = place_comment(placement, &self.locator);
let placement = place_comment(placement, self.comment_ranges, &self.locator);
match placement {
CommentPlacement::Leading { node, comment } => {
self.push_leading_comment(node, comment);
@ -603,9 +604,10 @@ impl<'a> PushComment<'a> for CommentsMapBuilder<'a> {
}
impl<'a> CommentsMapBuilder<'a> {
pub(crate) fn new(locator: Locator<'a>) -> Self {
pub(crate) fn new(locator: Locator<'a>, comment_ranges: &'a CommentRanges) -> Self {
Self {
comments: CommentsMap::default(),
comment_ranges,
locator,
}
}

48
crates/ruff_python_formatter/src/expression/binary_like.rs
View file

@ -8,6 +8,7 @@ use ruff_python_ast::{
Constant, Expr, ExprAttribute, ExprBinOp, ExprBoolOp, ExprCompare, ExprConstant, ExprUnaryOp,
UnaryOp,
};
use ruff_python_trivia::CommentRanges;
use ruff_python_trivia::{SimpleToken, SimpleTokenKind, SimpleTokenizer};
use ruff_text_size::{Ranged, TextRange};
@ -179,7 +180,13 @@ impl<'a> BinaryLike<'a> {
) {
let expression = operand.expression();
match expression {
Expr::BinOp(binary) if !is_expression_parenthesized(expression.into(), source) => {
Expr::BinOp(binary)
if !is_expression_parenthesized(
expression.into(),
comments.ranges(),
source,
) =>
{
let leading_comments = operand
.leading_binary_comments()
.unwrap_or_else(|| comments.leading(binary));
@ -198,7 +205,11 @@ impl<'a> BinaryLike<'a> {
);
}
Expr::Compare(compare)
if !is_expression_parenthesized(expression.into(), source) =>
if !is_expression_parenthesized(
expression.into(),
comments.ranges(),
source,
) =>
{
let leading_comments = operand
.leading_binary_comments()
@ -218,7 +229,11 @@ impl<'a> BinaryLike<'a> {
);
}
Expr::BoolOp(bool_op)
if !is_expression_parenthesized(expression.into(), source) =>
if !is_expression_parenthesized(
expression.into(),
comments.ranges(),
source,
) =>
{
let leading_comments = operand
.leading_binary_comments()
@ -282,7 +297,11 @@ impl Format<PyFormatContext<'_>> for BinaryLike<'_> {
AnyString::from_expression(operand.expression())
.filter(|string| {
string.is_implicit_concatenated()
&& !is_expression_parenthesized(string.into(), source)
&& !is_expression_parenthesized(
string.into(),
comments.ranges(),
source,
)
})
.map(|string| (index, string, operand))
})
@ -430,6 +449,7 @@ impl Format<PyFormatContext<'_>> for BinaryLike<'_> {
if (right_operand_has_leading_comments
&& !is_expression_parenthesized(
right_operand.expression().into(),
f.context().comments().ranges(),
f.context().source(),
))
|| right_operator.has_trailing_comments()
@ -466,11 +486,16 @@ impl Format<PyFormatContext<'_>> for BinaryLike<'_> {
}
}
fn is_simple_power_expression(left: &Expr, right: &Expr, source: &str) -> bool {
fn is_simple_power_expression(
left: &Expr,
right: &Expr,
comment_range: &CommentRanges,
source: &str,
) -> bool {
is_simple_power_operand(left)
&& is_simple_power_operand(right)
&& !is_expression_parenthesized(left.into(), source)
&& !is_expression_parenthesized(right.into(), source)
&& !is_expression_parenthesized(left.into(), comment_range, source)
&& !is_expression_parenthesized(right.into(), comment_range, source)
}
/// Return `true` if an [`Expr`] adheres to [Black's definition](https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#line-breaks-binary-operators)
@ -664,6 +689,7 @@ impl Format<PyFormatContext<'_>> for FlatBinaryExpressionSlice<'_> {
&& is_simple_power_expression(
left.last_operand().expression(),
right.first_operand().expression(),
f.context().comments().ranges(),
f.context().source(),
);
@ -806,7 +832,7 @@ impl<'a> Operand<'a> {
} => !leading_comments.is_empty(),
Operand::Middle { expression } | Operand::Right { expression, .. } => {
let leading = comments.leading(*expression);
if is_expression_parenthesized((*expression).into(), source) {
if is_expression_parenthesized((*expression).into(), comments.ranges(), source) {
leading.iter().any(|comment| {
!comment.is_formatted()
&& matches!(
@ -853,7 +879,11 @@ impl Format<PyFormatContext<'_>> for Operand<'_> {
fn fmt(&self, f: &mut Formatter<PyFormatContext<'_>>) -> FormatResult<()> {
let expression = self.expression();
return if is_expression_parenthesized(expression.into(), f.context().source()) {
return if is_expression_parenthesized(
expression.into(),
f.context().comments().ranges(),
f.context().source(),
) {
let comments = f.context().comments().clone();
let expression_comments = comments.leading_dangling_trailing(expression);

15
crates/ruff_python_formatter/src/expression/expr_attribute.rs
View file

@ -45,7 +45,7 @@ impl FormatNodeRule<ExprAttribute> for FormatExprAttribute {
value: Constant::Int(_) | Constant::Float(_),
..
})
) || is_expression_parenthesized(value.into(), f.context().source());
) || is_expression_parenthesized(value.into(), f.context().comments().ranges(), f.context().source());
if call_chain_layout == CallChainLayout::Fluent {
if parenthesize_value {
@ -142,15 +142,22 @@ impl NeedsParentheses for ExprAttribute {
context: &PyFormatContext,
) -> OptionalParentheses {
// Checks if there are any own line comments in an attribute chain (a.b.c).
if CallChainLayout::from_expression(self.into(), context.source())
== CallChainLayout::Fluent
if CallChainLayout::from_expression(
self.into(),
context.comments().ranges(),
context.source(),
) == CallChainLayout::Fluent
{
OptionalParentheses::Multiline
} else if context.comments().has_dangling(self) {
OptionalParentheses::Always
} else if self.value.is_name_expr() {
OptionalParentheses::BestFit
} else if is_expression_parenthesized(self.value.as_ref().into(), context.source()) {
} else if is_expression_parenthesized(
self.value.as_ref().into(),
context.comments().ranges(),
context.source(),
) {
OptionalParentheses::Never
} else {
self.value.needs_parentheses(self.into(), context)

13
crates/ruff_python_formatter/src/expression/expr_call.rs
View file

@ -82,13 +82,20 @@ impl NeedsParentheses for ExprCall {
_parent: AnyNodeRef,
context: &PyFormatContext,
) -> OptionalParentheses {
if CallChainLayout::from_expression(self.into(), context.source())
== CallChainLayout::Fluent
if CallChainLayout::from_expression(
self.into(),
context.comments().ranges(),
context.source(),
) == CallChainLayout::Fluent
{
OptionalParentheses::Multiline
} else if context.comments().has_dangling(self) {
OptionalParentheses::Always
} else if is_expression_parenthesized(self.func.as_ref().into(), context.source()) {
} else if is_expression_parenthesized(
self.func.as_ref().into(),
context.comments().ranges(),
context.source(),
) {
OptionalParentheses::Never
} else {
self.func.needs_parentheses(self.into(), context)

6
crates/ruff_python_formatter/src/expression/expr_if_exp.rs
View file

@ -101,7 +101,11 @@ impl Format<PyFormatContext<'_>> for FormatOrElse<'_> {
fn fmt(&self, f: &mut Formatter<PyFormatContext<'_>>) -> FormatResult<()> {
match self.orelse {
Expr::IfExp(expr)
if !is_expression_parenthesized(expr.into(), f.context().source()) =>
if !is_expression_parenthesized(
expr.into(),
f.context().comments().ranges(),
f.context().source(),
) =>
{
write!(f, [expr.format().with_options(ExprIfExpLayout::Nested)])
}

13
crates/ruff_python_formatter/src/expression/expr_subscript.rs
View file

@ -89,11 +89,18 @@ impl NeedsParentheses for ExprSubscript {
context: &PyFormatContext,
) -> OptionalParentheses {
{
if CallChainLayout::from_expression(self.into(), context.source())
== CallChainLayout::Fluent
if CallChainLayout::from_expression(
self.into(),
context.comments().ranges(),
context.source(),
) == CallChainLayout::Fluent
{
OptionalParentheses::Multiline
} else if is_expression_parenthesized(self.value.as_ref().into(), context.source()) {
} else if is_expression_parenthesized(
self.value.as_ref().into(),
context.comments().ranges(),
context.source(),
) {
OptionalParentheses::Never
} else {
match self.value.needs_parentheses(self.into(), context) {

12
crates/ruff_python_formatter/src/expression/expr_unary_op.rs
View file

@ -46,7 +46,11 @@ impl FormatNodeRule<ExprUnaryOp> for FormatExprUnaryOp {
// a)
// ```
if comments.has_leading(operand.as_ref())
&& !is_expression_parenthesized(operand.as_ref().into(), f.context().source())
&& !is_expression_parenthesized(
operand.as_ref().into(),
f.context().comments().ranges(),
f.context().source(),
)
{
hard_line_break().fmt(f)?;
} else if op.is_not() {
@ -72,7 +76,11 @@ impl NeedsParentheses for ExprUnaryOp {
context: &PyFormatContext,
) -> OptionalParentheses {
// We preserve the parentheses of the operand. It should not be necessary to break this expression.
if is_expression_parenthesized(self.operand.as_ref().into(), context.source()) {
if is_expression_parenthesized(
self.operand.as_ref().into(),
context.comments().ranges(),
context.source(),
) {
OptionalParentheses::Never
} else {
OptionalParentheses::Multiline

41
crates/ruff_python_formatter/src/expression/mod.rs
View file

@ -9,6 +9,7 @@ use ruff_python_ast as ast;
use ruff_python_ast::node::AnyNodeRef;
use ruff_python_ast::visitor::preorder::{walk_expr, PreorderVisitor};
use ruff_python_ast::{Constant, Expr, ExpressionRef, Operator};
use ruff_python_trivia::CommentRanges;
use crate::builders::parenthesize_if_expands;
use crate::comments::leading_comments;
@ -103,9 +104,11 @@ impl FormatRule<Expr, PyFormatContext<'_>> for FormatExpr {
});
let parenthesize = match parentheses {
Parentheses::Preserve => {
is_expression_parenthesized(expression.into(), f.context().source())
}
Parentheses::Preserve => is_expression_parenthesized(
expression.into(),
f.context().comments().ranges(),
f.context().source(),
),
Parentheses::Always => true,
// Fluent style means we already have parentheses
Parentheses::Never => false,
@ -186,7 +189,11 @@ impl Format<PyFormatContext<'_>> for MaybeParenthesizeExpression<'_> {
let comments = f.context().comments();
let preserve_parentheses = parenthesize.is_optional()
&& is_expression_parenthesized((*expression).into(), f.context().source());
&& is_expression_parenthesized(
(*expression).into(),
f.context().comments().ranges(),
f.context().source(),
);
let has_comments =
comments.has_leading(*expression) || comments.has_trailing_own_line(*expression);
@ -581,7 +588,11 @@ impl<'input> PreorderVisitor<'input> for CanOmitOptionalParenthesesVisitor<'inpu
self.last = Some(expr);
// Rule only applies for non-parenthesized expressions.
if is_expression_parenthesized(expr.into(), self.context.source()) {
if is_expression_parenthesized(
expr.into(),
self.context.comments().ranges(),
self.context.source(),
) {
self.any_parenthesized_expressions = true;
} else {
self.visit_subexpression(expr);
@ -635,7 +646,11 @@ pub enum CallChainLayout {
}
impl CallChainLayout {
pub(crate) fn from_expression(mut expr: ExpressionRef, source: &str) -> Self {
pub(crate) fn from_expression(
mut expr: ExpressionRef,
comment_ranges: &CommentRanges,
source: &str,
) -> Self {
let mut attributes_after_parentheses = 0;
loop {
match expr {
@ -646,7 +661,7 @@ impl CallChainLayout {
// data[:100].T
// ^^^^^^^^^^ value
// ```
if is_expression_parenthesized(value.into(), source) {
if is_expression_parenthesized(value.into(), comment_ranges, source) {
// `(a).b`. We preserve these parentheses so don't recurse
attributes_after_parentheses += 1;
break;
@ -674,7 +689,7 @@ impl CallChainLayout {
// f2 = (a).w().t(1,)
// ^ expr
// ```
if is_expression_parenthesized(expr, source) {
if is_expression_parenthesized(expr, comment_ranges, source) {
attributes_after_parentheses += 1;
}
@ -683,7 +698,7 @@ impl CallChainLayout {
}
// We preserve these parentheses so don't recurse
if is_expression_parenthesized(expr, source) {
if is_expression_parenthesized(expr, comment_ranges, source) {
break;
}
}
@ -704,7 +719,11 @@ impl CallChainLayout {
match self {
CallChainLayout::Default => {
if f.context().node_level().is_parenthesized() {
CallChainLayout::from_expression(item.into(), f.context().source())
CallChainLayout::from_expression(
item.into(),
f.context().comments().ranges(),
f.context().source(),
)
} else {
CallChainLayout::NonFluent
}
@ -745,7 +764,7 @@ fn has_parentheses(expr: &Expr, context: &PyFormatContext) -> Option<OwnParenthe
// Otherwise, if the node lacks parentheses (e.g., `(1)`) or only contains empty parentheses
// (e.g., `([])`), we need to check for surrounding parentheses.
if is_expression_parenthesized(expr.into(), context.source()) {
if is_expression_parenthesized(expr.into(), context.comments().ranges(), context.source()) {
return Some(OwnParentheses::NonEmpty);
}

20
crates/ruff_python_formatter/src/expression/parentheses.rs
View file

@ -2,7 +2,10 @@ use ruff_formatter::prelude::tag::Condition;
use ruff_formatter::{format_args, write, Argument, Arguments, FormatContext, FormatOptions};
use ruff_python_ast::node::AnyNodeRef;
use ruff_python_ast::ExpressionRef;
use ruff_python_trivia::{first_non_trivia_token, SimpleToken, SimpleTokenKind, SimpleTokenizer};
use ruff_python_trivia::CommentRanges;
use ruff_python_trivia::{
first_non_trivia_token, BackwardsTokenizer, SimpleToken, SimpleTokenKind,
};
use ruff_text_size::Ranged;
use crate::comments::{
@ -100,7 +103,11 @@ pub enum Parentheses {
Never,
}
pub(crate) fn is_expression_parenthesized(expr: ExpressionRef, contents: &str) -> bool {
pub(crate) fn is_expression_parenthesized(
expr: ExpressionRef,
comment_ranges: &CommentRanges,
contents: &str,
) -> bool {
// First test if there's a closing parentheses because it tends to be cheaper.
if matches!(
first_non_trivia_token(expr.end(), contents),
@ -109,11 +116,10 @@ pub(crate) fn is_expression_parenthesized(expr: ExpressionRef, contents: &str) -
..
})
) {
let mut tokenizer =
SimpleTokenizer::up_to_without_back_comment(expr.start(), contents).skip_trivia();
matches!(
tokenizer.next_back(),
BackwardsTokenizer::up_to(expr.start(), contents, comment_ranges)
.skip_trivia()
.next(),
Some(SimpleToken {
kind: SimpleTokenKind::LParen,
..
@ -418,6 +424,7 @@ impl Format<PyFormatContext<'_>> for FormatEmptyParenthesized<'_> {
mod tests {
use ruff_python_ast::ExpressionRef;
use ruff_python_parser::parse_expression;
use ruff_python_trivia::CommentRanges;
use crate::expression::parentheses::is_expression_parenthesized;
@ -427,6 +434,7 @@ mod tests {
let expr = parse_expression(expression, "<filename>").unwrap();
assert!(!is_expression_parenthesized(
ExpressionRef::from(&expr),
&CommentRanges::default(),
expression
));
}

5
crates/ruff_python_formatter/src/lib.rs
View file

@ -5,9 +5,10 @@ use ruff_formatter::prelude::*;
use ruff_formatter::{format, FormatError, Formatted, PrintError, Printed, SourceCode};
use ruff_python_ast::node::AstNode;
use ruff_python_ast::Mod;
use ruff_python_index::{CommentRanges, CommentRangesBuilder};
use ruff_python_index::CommentRangesBuilder;
use ruff_python_parser::lexer::{lex, LexicalError};
use ruff_python_parser::{parse_tokens, Mode, ParseError};
use ruff_python_trivia::CommentRanges;
use ruff_source_file::Locator;
use crate::comments::{
@ -120,7 +121,7 @@ impl From<ParseError> for FormatModuleError {
}
}
#[tracing::instrument(level=Level::TRACE, skip_all)]
#[tracing::instrument(level = Level::TRACE, skip_all)]
pub fn format_module(
contents: &str,
options: PyFormatOptions,

24
crates/ruff_python_formatter/src/pattern/mod.rs
View file

@ -1,7 +1,10 @@
use ruff_formatter::{FormatOwnedWithRule, FormatRefWithRule, FormatRule, FormatRuleWithOptions};
use ruff_python_ast::node::AnyNodeRef;
use ruff_python_ast::Pattern;
use ruff_python_trivia::{first_non_trivia_token, SimpleToken, SimpleTokenKind, SimpleTokenizer};
use ruff_python_trivia::CommentRanges;
use ruff_python_trivia::{
first_non_trivia_token, BackwardsTokenizer, SimpleToken, SimpleTokenKind,
};
use ruff_text_size::Ranged;
use crate::expression::parentheses::{
@ -48,7 +51,11 @@ impl FormatRule<Pattern, PyFormatContext<'_>> for FormatPattern {
});
let parenthesize = match self.parentheses {
Parentheses::Preserve => is_pattern_parenthesized(pattern, f.context().source()),
Parentheses::Preserve => is_pattern_parenthesized(
pattern,
f.context().comments().ranges(),
f.context().source(),
),
Parentheses::Always => true,
Parentheses::Never => false,
};
@ -98,7 +105,11 @@ impl<'ast> IntoFormat<PyFormatContext<'ast>> for Pattern {
}
}
fn is_pattern_parenthesized(pattern: &Pattern, contents: &str) -> bool {
fn is_pattern_parenthesized(
pattern: &Pattern,
comment_ranges: &CommentRanges,
contents: &str,
) -> bool {
// First test if there's a closing parentheses because it tends to be cheaper.
if matches!(
first_non_trivia_token(pattern.end(), contents),
@ -107,11 +118,10 @@ fn is_pattern_parenthesized(pattern: &Pattern, contents: &str) -> bool {
..
})
) {
let mut tokenizer =
SimpleTokenizer::up_to_without_back_comment(pattern.start(), contents).skip_trivia();
matches!(
tokenizer.next_back(),
BackwardsTokenizer::up_to(pattern.start(), contents, comment_ranges)
.skip_trivia()
.next(),
Some(SimpleToken {
kind: SimpleTokenKind::LParen,
..

8
crates/ruff_python_formatter/src/statement/suite.rs
View file

@ -561,6 +561,7 @@ impl Format<PyFormatContext<'_>> for SuiteChildStatement<'_> {
mod tests {
use ruff_formatter::format;
use ruff_python_parser::parse_suite;
use ruff_python_trivia::CommentRanges;
use crate::comments::Comments;
use crate::prelude::*;
@ -591,7 +592,12 @@ def trailing_func():
let statements = parse_suite(source, "test.py").unwrap();
let context = PyFormatContext::new(PyFormatOptions::default(), source, Comments::default());
let comment_ranges = CommentRanges::default();
let context = PyFormatContext::new(
PyFormatOptions::default(),
source,
Comments::from_ranges(&comment_ranges),
);
let test_formatter =
format_with(|f: &mut PyFormatter| statements.format().with_options(level).fmt(f));