mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-09-25 19:39:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Join implicit concatenated strings when they fit on a line (#13663)

Browse source
This commit is contained in:
Micha Reiser 2024-10-24 11:52:22 +02:00 • committed by GitHub
parent e402e27a09
commit 73ee72b665
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
50 changed files with 3907 additions and 363 deletions
Download patch file Download diff file Expand all files Collapse all files

35
crates/ruff_python_formatter/src/comments/mod.rs
View file

@ -431,6 +431,41 @@ impl<'a> Comments<'a> {
pub(crate) fn debug(&'a self, source_code: SourceCode<'a>) -> DebugComments<'a> {
DebugComments::new(&self.data.comments, source_code)
}
/// Returns true if the node itself or any of its descendants have comments.
pub(crate) fn contains_comments(&self, node: AnyNodeRef) -> bool {
use ruff_python_ast::visitor::source_order::{SourceOrderVisitor, TraversalSignal};
struct Visitor<'a> {
comments: &'a Comments<'a>,
has_comment: bool,
}
impl<'a> SourceOrderVisitor<'a> for Visitor<'a> {
fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal {
if self.has_comment {
TraversalSignal::Skip
} else if self.comments.has(node) {
self.has_comment = true;
TraversalSignal::Skip
} else {
TraversalSignal::Traverse
}
}
}
if self.has(node) {
return true;
}
let mut visitor = Visitor {
comments: self,
has_comment: false,
};
node.visit_preorder(&mut visitor);
visitor.has_comment
}
}
pub(crate) type LeadingDanglingTrailingComments<'a> = LeadingDanglingTrailing<'a, SourceComment>;

2
crates/ruff_python_formatter/src/context.rs
View file

@ -8,7 +8,6 @@ use ruff_source_file::Locator;
use std::fmt::{Debug, Formatter};
use std::ops::{Deref, DerefMut};
#[derive(Clone)]
pub struct PyFormatContext<'a> {
options: PyFormatOptions,
contents: &'a str,
@ -52,7 +51,6 @@ impl<'a> PyFormatContext<'a> {
self.contents
}
#[allow(unused)]
pub(crate) fn locator(&self) -> Locator<'a> {
Locator::new(self.contents)
}

2
crates/ruff_python_formatter/src/expression/binary_like.rs
View file

@ -20,7 +20,7 @@ use crate::expression::parentheses::{
};
use crate::expression::OperatorPrecedence;
use crate::prelude::*;
use crate::string::FormatImplicitConcatenatedString;
use crate::string::implicit::FormatImplicitConcatenatedString;
#[derive(Copy, Clone, Debug)]
pub(super) enum BinaryLike<'a> {

19
crates/ruff_python_formatter/src/expression/expr_bytes_literal.rs
View file

@ -5,7 +5,8 @@ use crate::expression::parentheses::{
in_parentheses_only_group, NeedsParentheses, OptionalParentheses,
};
use crate::prelude::*;
use crate::string::{FormatImplicitConcatenatedString, StringLikeExtensions};
use crate::string::implicit::FormatImplicitConcatenatedStringFlat;
use crate::string::{implicit::FormatImplicitConcatenatedString, StringLikeExtensions};
#[derive(Default)]
pub struct FormatExprBytesLiteral;
@ -14,9 +15,19 @@ impl FormatNodeRule<ExprBytesLiteral> for FormatExprBytesLiteral {
fn fmt_fields(&self, item: &ExprBytesLiteral, f: &mut PyFormatter) -> FormatResult<()> {
let ExprBytesLiteral { value, .. } = item;
match value.as_slice() {
[bytes_literal] => bytes_literal.format().fmt(f),
_ => in_parentheses_only_group(&FormatImplicitConcatenatedString::new(item)).fmt(f),
if let [bytes_literal] = value.as_slice() {
bytes_literal.format().fmt(f)
} else {
// Always join byte literals that aren't parenthesized and thus, always on a single line.
if !f.context().node_level().is_parenthesized() {
if let Some(format_flat) =
FormatImplicitConcatenatedStringFlat::new(item.into(), f.context())
{
return format_flat.fmt(f);
}
}
in_parentheses_only_group(&FormatImplicitConcatenatedString::new(item)).fmt(f)
}
}
}

24
crates/ruff_python_formatter/src/expression/expr_f_string.rs
View file

@ -7,7 +7,8 @@ use crate::expression::parentheses::{
};
use crate::other::f_string_part::FormatFStringPart;
use crate::prelude::*;
use crate::string::{FormatImplicitConcatenatedString, Quoting, StringLikeExtensions};
use crate::string::implicit::FormatImplicitConcatenatedStringFlat;
use crate::string::{implicit::FormatImplicitConcatenatedString, Quoting, StringLikeExtensions};
#[derive(Default)]
pub struct FormatExprFString;
@ -16,13 +17,23 @@ impl FormatNodeRule<ExprFString> for FormatExprFString {
fn fmt_fields(&self, item: &ExprFString, f: &mut PyFormatter) -> FormatResult<()> {
let ExprFString { value, .. } = item;
match value.as_slice() {
[f_string_part] => FormatFStringPart::new(
if let [f_string_part] = value.as_slice() {
FormatFStringPart::new(
f_string_part,
f_string_quoting(item, &f.context().locator()),
)
.fmt(f),
_ => in_parentheses_only_group(&FormatImplicitConcatenatedString::new(item)).fmt(f),
.fmt(f)
} else {
// Always join fstrings that aren't parenthesized and thus, are always on a single line.
if !f.context().node_level().is_parenthesized() {
if let Some(format_flat) =
FormatImplicitConcatenatedStringFlat::new(item.into(), f.context())
{
return format_flat.fmt(f);
}
}
in_parentheses_only_group(&FormatImplicitConcatenatedString::new(item)).fmt(f)
}
}
}
@ -35,6 +46,7 @@ impl NeedsParentheses for ExprFString {
) -> OptionalParentheses {
if self.value.is_implicit_concatenated() {
OptionalParentheses::Multiline
}
// TODO(dhruvmanila): Ideally what we want here is a new variant which
// is something like:
// - If the expression fits by just adding the parentheses, then add them and
@ -53,7 +65,7 @@ impl NeedsParentheses for ExprFString {
// ```
// This isn't decided yet, refer to the relevant discussion:
// https://github.com/astral-sh/ruff/discussions/9785
} else if StringLike::FString(self).is_multiline(context.source()) {
else if StringLike::FString(self).is_multiline(context.source()) {
OptionalParentheses::Never
} else {
OptionalParentheses::BestFit

25
crates/ruff_python_formatter/src/expression/expr_string_literal.rs
View file

@ -6,7 +6,8 @@ use crate::expression::parentheses::{
};
use crate::other::string_literal::StringLiteralKind;
use crate::prelude::*;
use crate::string::{FormatImplicitConcatenatedString, StringLikeExtensions};
use crate::string::implicit::FormatImplicitConcatenatedStringFlat;
use crate::string::{implicit::FormatImplicitConcatenatedString, StringLikeExtensions};
#[derive(Default)]
pub struct FormatExprStringLiteral {
@ -26,16 +27,20 @@ impl FormatNodeRule<ExprStringLiteral> for FormatExprStringLiteral {
fn fmt_fields(&self, item: &ExprStringLiteral, f: &mut PyFormatter) -> FormatResult<()> {
let ExprStringLiteral { value, .. } = item;
match value.as_slice() {
[string_literal] => string_literal.format().with_options(self.kind).fmt(f),
_ => {
// This is just a sanity check because [`DocstringStmt::try_from_statement`]
// ensures that the docstring is a *single* string literal.
assert!(!self.kind.is_docstring());
in_parentheses_only_group(&FormatImplicitConcatenatedString::new(item))
if let [string_literal] = value.as_slice() {
string_literal.format().with_options(self.kind).fmt(f)
} else {
// Always join strings that aren't parenthesized and thus, always on a single line.
if !f.context().node_level().is_parenthesized() {
if let Some(mut format_flat) =
FormatImplicitConcatenatedStringFlat::new(item.into(), f.context())
{
format_flat.set_docstring(self.kind.is_docstring());
return format_flat.fmt(f);
}
}
.fmt(f),
in_parentheses_only_group(&FormatImplicitConcatenatedString::new(item)).fmt(f)
}
}
}

59
crates/ruff_python_formatter/src/expression/mod.rs
View file

@ -21,6 +21,7 @@ use crate::expression::parentheses::{
use crate::prelude::*;
use crate::preview::{
is_empty_parameters_no_unnecessary_parentheses_around_return_value_enabled,
is_f_string_implicit_concatenated_string_literal_quotes_enabled,
is_hug_parens_with_braces_and_square_brackets_enabled,
};
@ -405,38 +406,39 @@ impl Format<PyFormatContext<'_>> for MaybeParenthesizeExpression<'_> {
needs_parentheses => needs_parentheses,
};
let unparenthesized = expression.format().with_options(Parentheses::Never);
match needs_parentheses {
OptionalParentheses::Multiline => match parenthesize {
Parenthesize::IfBreaksParenthesized | Parenthesize::IfBreaksParenthesizedNested if !is_empty_parameters_no_unnecessary_parentheses_around_return_value_enabled(f.context()) => {
parenthesize_if_expands(&expression.format().with_options(Parentheses::Never))
.fmt(f)
}
Parenthesize::IfRequired => {
expression.format().with_options(Parentheses::Never).fmt(f)
parenthesize_if_expands(&unparenthesized).fmt(f)
}
Parenthesize::IfRequired => unparenthesized.fmt(f),
Parenthesize::Optional | Parenthesize::IfBreaks | Parenthesize::IfBreaksParenthesized | Parenthesize::IfBreaksParenthesizedNested => {
if can_omit_optional_parentheses(expression, f.context()) {
optional_parentheses(&expression.format().with_options(Parentheses::Never))
.fmt(f)
optional_parentheses(&unparenthesized).fmt(f)
} else {
parenthesize_if_expands(
&expression.format().with_options(Parentheses::Never),
)
.fmt(f)
parenthesize_if_expands(&unparenthesized).fmt(f)
}
}
},
OptionalParentheses::BestFit => match parenthesize {
Parenthesize::IfBreaksParenthesized | Parenthesize::IfBreaksParenthesizedNested if !is_empty_parameters_no_unnecessary_parentheses_around_return_value_enabled(f.context()) =>
parenthesize_if_expands(&unparenthesized).fmt(f),
Parenthesize::IfBreaksParenthesized | Parenthesize::IfBreaksParenthesizedNested => {
parenthesize_if_expands(&expression.format().with_options(Parentheses::Never))
.fmt(f)
// Can-omit layout is relevant for `"abcd".call`. We don't want to add unnecessary
// parentheses in this case.
if can_omit_optional_parentheses(expression, f.context()) {
optional_parentheses(&unparenthesized).fmt(f)
} else {
parenthesize_if_expands(&unparenthesized).fmt(f)
}
}
Parenthesize::Optional | Parenthesize::IfRequired => {
expression.format().with_options(Parentheses::Never).fmt(f)
}
Parenthesize::Optional | Parenthesize::IfRequired => unparenthesized.fmt(f),
Parenthesize::IfBreaks => {
if node_comments.has_trailing() {
@ -446,7 +448,7 @@ impl Format<PyFormatContext<'_>> for MaybeParenthesizeExpression<'_> {
let group_id = f.group_id("optional_parentheses");
let f = &mut WithNodeLevel::new(NodeLevel::Expression(Some(group_id)), f);
best_fit_parenthesize(&expression.format().with_options(Parentheses::Never))
best_fit_parenthesize(&unparenthesized)
.with_group_id(Some(group_id))
.fmt(f)
}
@ -454,13 +456,13 @@ impl Format<PyFormatContext<'_>> for MaybeParenthesizeExpression<'_> {
},
OptionalParentheses::Never => match parenthesize {
Parenthesize::IfBreaksParenthesized | Parenthesize::IfBreaksParenthesizedNested if !is_empty_parameters_no_unnecessary_parentheses_around_return_value_enabled(f.context()) => {
parenthesize_if_expands(&expression.format().with_options(Parentheses::Never))
parenthesize_if_expands(&unparenthesized)
.with_indent(!is_expression_huggable(expression, f.context()))
.fmt(f)
}
Parenthesize::Optional | Parenthesize::IfBreaks | Parenthesize::IfRequired | Parenthesize::IfBreaksParenthesized | Parenthesize::IfBreaksParenthesizedNested => {
expression.format().with_options(Parentheses::Never).fmt(f)
unparenthesized.fmt(f)
}
},
@ -768,15 +770,26 @@ impl<'input> CanOmitOptionalParenthesesVisitor<'input> {
Expr::StringLiteral(ast::ExprStringLiteral { value, .. })
if value.is_implicit_concatenated() =>
{
self.update_max_precedence(OperatorPrecedence::String);
if !is_f_string_implicit_concatenated_string_literal_quotes_enabled(self.context) {
self.update_max_precedence(OperatorPrecedence::String);
}
return;
}
Expr::BytesLiteral(ast::ExprBytesLiteral { value, .. })
if value.is_implicit_concatenated() =>
{
self.update_max_precedence(OperatorPrecedence::String);
if !is_f_string_implicit_concatenated_string_literal_quotes_enabled(self.context) {
self.update_max_precedence(OperatorPrecedence::String);
}
return;
}
Expr::FString(ast::ExprFString { value, .. }) if value.is_implicit_concatenated() => {
self.update_max_precedence(OperatorPrecedence::String);
if !is_f_string_implicit_concatenated_string_literal_quotes_enabled(self.context) {
self.update_max_precedence(OperatorPrecedence::String);
}
return;
}

15
crates/ruff_python_formatter/src/other/f_string.rs
View file

@ -76,14 +76,9 @@ impl Format<PyFormatContext<'_>> for FormatFString<'_> {
let quotes = StringQuotes::from(string_kind);
write!(f, [string_kind.prefix(), quotes])?;
f.join()
.entries(
self.value
.elements
.iter()
.map(|element| FormatFStringElement::new(element, context)),
)
.finish()?;
for element in &self.value.elements {
FormatFStringElement::new(element, context).fmt(f)?;
}
// Ending quote
quotes.fmt(f)
@ -98,7 +93,7 @@ pub(crate) struct FStringContext {
}
impl FStringContext {
const fn new(flags: AnyStringFlags, layout: FStringLayout) -> Self {
pub(crate) const fn new(flags: AnyStringFlags, layout: FStringLayout) -> Self {
Self {
enclosing_flags: flags,
layout,
@ -125,7 +120,7 @@ pub(crate) enum FStringLayout {
}
impl FStringLayout {
fn from_f_string(f_string: &FString, locator: &Locator) -> Self {
pub(crate) fn from_f_string(f_string: &FString, locator: &Locator) -> Self {
// Heuristic: Allow breaking the f-string expressions across multiple lines
// only if there already is at least one multiline expression. This puts the
// control in the hands of the user to decide if they want to break the

11
crates/ruff_python_formatter/src/other/f_string_element.rs
View file

@ -61,7 +61,8 @@ impl<'a> FormatFStringLiteralElement<'a> {
impl Format<PyFormatContext<'_>> for FormatFStringLiteralElement<'_> {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let literal_content = f.context().locator().slice(self.element.range());
let normalized = normalize_string(literal_content, 0, self.fstring_flags, true);
let normalized =
normalize_string(literal_content, 0, self.fstring_flags, false, false, true);
match &normalized {
Cow::Borrowed(_) => source_text_slice(self.element.range()).fmt(f),
Cow::Owned(normalized) => text(normalized).fmt(f),
@ -235,11 +236,9 @@ impl Format<PyFormatContext<'_>> for FormatFStringExpressionElement<'_> {
if let Some(format_spec) = format_spec.as_deref() {
token(":").fmt(f)?;
f.join()
.entries(format_spec.elements.iter().map(|element| {
FormatFStringElement::new(element, self.context.f_string())
}))
.finish()?;
for element in &format_spec.elements {
FormatFStringElement::new(element, self.context.f_string()).fmt(f)?;
}
// These trailing comments can only occur if the format specifier is
// present. For example,

39
crates/ruff_python_formatter/src/pattern/mod.rs
View file

@ -14,6 +14,7 @@ use crate::expression::parentheses::{
optional_parentheses, parenthesized, NeedsParentheses, OptionalParentheses, Parentheses,
};
use crate::prelude::*;
use crate::preview::is_join_implicit_concatenated_string_enabled;
pub(crate) mod pattern_arguments;
pub(crate) mod pattern_keyword;
@ -226,7 +227,7 @@ pub(crate) fn can_pattern_omit_optional_parentheses(
pattern: &Pattern,
context: &PyFormatContext,
) -> bool {
let mut visitor = CanOmitOptionalParenthesesVisitor::default();
let mut visitor = CanOmitOptionalParenthesesVisitor::new(context);
visitor.visit_pattern(pattern, context);
if !visitor.any_parenthesized_expressions {
@ -271,16 +272,32 @@ pub(crate) fn can_pattern_omit_optional_parentheses(
}
}
#[derive(Debug, Default)]
#[derive(Debug)]
struct CanOmitOptionalParenthesesVisitor<'input> {
max_precedence: OperatorPrecedence,
max_precedence_count: usize,
any_parenthesized_expressions: bool,
join_implicit_concatenated_strings: bool,
last: Option<&'input Pattern>,
first: First<'input>,
}
impl<'a> CanOmitOptionalParenthesesVisitor<'a> {
fn new(context: &PyFormatContext) -> Self {
Self {
max_precedence: OperatorPrecedence::default(),
max_precedence_count: 0,
any_parenthesized_expressions: false,
// TODO: Derive default for `CanOmitOptionalParenthesesVisitor` when removing the `join_implicit_concatenated_strings`
// preview style.
join_implicit_concatenated_strings: is_join_implicit_concatenated_string_enabled(
context,
),
last: None,
first: First::default(),
}
}
fn visit_pattern(&mut self, pattern: &'a Pattern, context: &PyFormatContext) {
match pattern {
Pattern::MatchSequence(_) | Pattern::MatchMapping(_) => {
@ -289,18 +306,24 @@ impl<'a> CanOmitOptionalParenthesesVisitor<'a> {
Pattern::MatchValue(value) => match &*value.value {
Expr::StringLiteral(string) => {
self.update_max_precedence(OperatorPrecedence::String, string.value.len());
if !self.join_implicit_concatenated_strings {
self.update_max_precedence(OperatorPrecedence::String, string.value.len());
}
}
Expr::BytesLiteral(bytes) => {
self.update_max_precedence(OperatorPrecedence::String, bytes.value.len());
if !self.join_implicit_concatenated_strings {
self.update_max_precedence(OperatorPrecedence::String, bytes.value.len());
}
}
// F-strings are allowed according to python's grammar but fail with a syntax error at runtime.
// That's why we need to support them for formatting.
Expr::FString(string) => {
self.update_max_precedence(
OperatorPrecedence::String,
string.value.as_slice().len(),
);
if !self.join_implicit_concatenated_strings {
self.update_max_precedence(
OperatorPrecedence::String,
string.value.as_slice().len(),
);
}
}
Expr::NumberLiteral(_) | Expr::Attribute(_) | Expr::UnaryOp(_) => {

8
crates/ruff_python_formatter/src/preview.rs
View file

@ -62,3 +62,11 @@ pub(crate) fn is_docstring_code_block_in_docstring_indent_enabled(
) -> bool {
context.is_preview()
}
/// Returns `true` if implicitly concatenated strings should be joined if they all fit on a single line.
/// See [#9457](https://github.com/astral-sh/ruff/issues/9457)
/// WARNING: This preview style depends on `is_empty_parameters_no_unnecessary_parentheses_around_return_value_enabled`
/// because it relies on the new semantic of `IfBreaksParenthesized`.
pub(crate) fn is_join_implicit_concatenated_string_enabled(context: &PyFormatContext) -> bool {
context.is_preview()
}

6
crates/ruff_python_formatter/src/range.rs
View file

@ -211,9 +211,9 @@ impl<'ast> SourceOrderVisitor<'ast> for FindEnclosingNode<'_, 'ast> {
// Don't pick potential docstrings as the closest enclosing node because `suite.rs` than fails to identify them as
// docstrings and docstring formatting won't kick in.
// Format the enclosing node instead and slice the formatted docstring from the result.
let is_maybe_docstring = node.as_stmt_expr().is_some_and(|stmt| {
DocstringStmt::is_docstring_statement(stmt, self.context.options().source_type())
});
let is_maybe_docstring = node
.as_stmt_expr()
.is_some_and(|stmt| DocstringStmt::is_docstring_statement(stmt, self.context));
if is_maybe_docstring {
return TraversalSignal::Skip;

9
crates/ruff_python_formatter/src/statement/stmt_assert.rs
View file

@ -6,6 +6,7 @@ use crate::comments::SourceComment;
use crate::expression::maybe_parenthesize_expression;
use crate::expression::parentheses::Parenthesize;
use crate::preview::is_join_implicit_concatenated_string_enabled;
use crate::{has_skip_comment, prelude::*};
#[derive(Default)]
@ -29,12 +30,18 @@ impl FormatNodeRule<StmtAssert> for FormatStmtAssert {
)?;
if let Some(msg) = msg {
let parenthesize = if is_join_implicit_concatenated_string_enabled(f.context()) {
Parenthesize::IfBreaksParenthesized
} else {
Parenthesize::IfBreaks
};
write!(
f,
[
token(","),
space(),
maybe_parenthesize_expression(msg, item, Parenthesize::IfBreaks),
maybe_parenthesize_expression(msg, item, parenthesize),
]
)?;
}

405
crates/ruff_python_formatter/src/statement/stmt_assign.rs
View file

@ -1,6 +1,6 @@
use ruff_formatter::{format_args, write, FormatError};
use ruff_formatter::{format_args, write, FormatError, RemoveSoftLinesBuffer};
use ruff_python_ast::{
AnyNodeRef, Expr, ExprAttribute, ExprCall, Operator, StmtAssign, TypeParams,
AnyNodeRef, Expr, ExprAttribute, ExprCall, Operator, StmtAssign, StringLike, TypeParams,
};
use crate::builders::parenthesize_if_expands;
@ -16,7 +16,11 @@ use crate::expression::{
can_omit_optional_parentheses, has_own_parentheses, has_parentheses,
maybe_parenthesize_expression,
};
use crate::preview::is_join_implicit_concatenated_string_enabled;
use crate::statement::trailing_semicolon;
use crate::string::implicit::{
FormatImplicitConcatenatedStringExpanded, FormatImplicitConcatenatedStringFlat,
};
use crate::{has_skip_comment, prelude::*};
#[derive(Default)]
@ -281,8 +285,11 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
match self {
FormatStatementsLastExpression::LeftToRight { value, statement } => {
let can_inline_comment = should_inline_comments(value, *statement, f.context());
let format_implicit_flat = StringLike::try_from(*value).ok().and_then(|string| {
FormatImplicitConcatenatedStringFlat::new(string, f.context())
});
if !can_inline_comment {
if !can_inline_comment && format_implicit_flat.is_none() {
return maybe_parenthesize_expression(
value,
*statement,
@ -301,28 +308,149 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
) {
let group_id = f.group_id("optional_parentheses");
let f = &mut WithNodeLevel::new(NodeLevel::Expression(Some(group_id)), f);
best_fit_parenthesize(&format_with(|f| {
// Special case for implicit concatenated strings in assignment value positions.
// The special handling is necessary to prevent an instability where an assignment has
// a trailing own line comment and the implicit concatenated string fits on the line,
// but only if the comment doesn't get inlined.
//
// ```python
// ____aaa = (
// "aaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbvvvvvvvvvvvvvvv"
// ) # c
// ```
//
// Without the special handling, this would get formatted to:
// ```python
// ____aaa = (
// "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbvvvvvvvvvvvvvvv"
// ) # c
// ```
//
// However, this now gets reformatted again because Ruff now takes the `BestFit` layout for the string
// because the value is no longer an implicit concatenated string.
// ```python
// ____aaa = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbvvvvvvvvvvvvvvv" # c
// ```
//
// The special handling here ensures that the implicit concatenated string only gets
// joined **if** it fits with the trailing comment inlined. Otherwise, keep the multiline
// formatting.
if let Some(flat) = format_implicit_flat {
inline_comments.mark_formatted();
let string = flat.string();
value.format().with_options(Parentheses::Never).fmt(f)?;
let flat = format_with(|f| {
if string.is_fstring() {
let mut buffer = RemoveSoftLinesBuffer::new(&mut *f);
if !inline_comments.is_empty() {
// If the expressions exceeds the line width, format the comments in the parentheses
if_group_breaks(&inline_comments).fmt(f)?;
write!(buffer, [flat])
} else {
flat.fmt(f)
}
})
.memoized();
// F-String containing an expression with a magic trailing comma, a comment, or a
// multiline debug expression should never be joined. Use the default layout.
// ```python
// aaaa = f"abcd{[
// 1,
// 2,
// ]}" "more"
// ```
if string.is_fstring() && flat.inspect(f)?.will_break() {
inline_comments.mark_unformatted();
return write!(
f,
[maybe_parenthesize_expression(
value,
*statement,
Parenthesize::IfBreaks,
)]
);
}
Ok(())
}))
.with_group_id(Some(group_id))
.fmt(f)?;
let expanded = format_with(|f| {
let f =
&mut WithNodeLevel::new(NodeLevel::Expression(Some(group_id)), f);
if !inline_comments.is_empty() {
// If the line fits into the line width, format the comments after the parenthesized expression
if_group_fits_on_line(&inline_comments)
write!(f, [FormatImplicitConcatenatedStringExpanded::new(string)])
});
// Join the implicit concatenated string if it fits on a single line
// ```python
// a = "testmorelong" # comment
// ```
let single_line = format_with(|f| write!(f, [flat, inline_comments]));
// Parenthesize the string but join the implicit concatenated string and inline the comment.
// ```python
// a = (
// "testmorelong" # comment
// )
// ```
let joined_parenthesized = format_with(|f| {
group(&format_args![
token("("),
soft_block_indent(&format_args![flat, inline_comments]),
token(")"),
])
.with_group_id(Some(group_id))
.should_expand(true)
.fmt(f)
});
// Keep the implicit concatenated string multiline and don't inline the comment.
// ```python
// a = (
// "test"
// "more"
// "long"
// ) # comment
// ```
let implicit_expanded = format_with(|f| {
group(&format_args![
token("("),
block_indent(&expanded),
token(")"),
inline_comments,
])
.with_group_id(Some(group_id))
.should_expand(true)
.fmt(f)
});
// We can't use `optional_parentheses` here because the `inline_comments` contains
// a `expand_parent` which results in an instability because the next format
// collapses the parentheses.
// We can't use `parenthesize_if_expands` because it defaults to
// the *flat* layout when the expanded layout doesn't fit.
best_fitting![single_line, joined_parenthesized, implicit_expanded]
.with_mode(BestFittingMode::AllLines)
.fmt(f)?;
} else {
best_fit_parenthesize(&format_once(|f| {
inline_comments.mark_formatted();
value.format().with_options(Parentheses::Never).fmt(f)?;
if !inline_comments.is_empty() {
// If the expressions exceeds the line width, format the comments in the parentheses
if_group_breaks(&inline_comments).fmt(f)?;
}
Ok(())
}))
.with_group_id(Some(group_id))
.fmt(f)?;
if !inline_comments.is_empty() {
// If the line fits into the line width, format the comments after the parenthesized expression
if_group_fits_on_line(&inline_comments)
.with_group_id(Some(group_id))
.fmt(f)?;
}
}
Ok(())
@ -339,10 +467,14 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
statement,
} => {
let should_inline_comments = should_inline_comments(value, *statement, f.context());
let format_implicit_flat = StringLike::try_from(*value).ok().and_then(|string| {
FormatImplicitConcatenatedStringFlat::new(string, f.context())
});
// Use the normal `maybe_parenthesize_layout` for splittable `value`s.
if !should_inline_comments
&& !should_non_inlineable_use_best_fit(value, *statement, f.context())
&& format_implicit_flat.is_none()
{
return write!(
f,
@ -364,7 +496,7 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
let expression_comments = comments.leading_dangling_trailing(*value);
// Don't inline comments for attribute and call expressions for black compatibility
let inline_comments = if should_inline_comments {
let inline_comments = if should_inline_comments || format_implicit_flat.is_some() {
OptionalParenthesesInlinedComments::new(
&expression_comments,
*statement,
@ -396,13 +528,14 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
// Prevent inline comments to be formatted as part of the expression.
inline_comments.mark_formatted();
let mut last_target = before_operator.memoized();
let last_target = before_operator.memoized();
let last_target_breaks = last_target.inspect(f)?.will_break();
// Don't parenthesize the `value` if it is known that the target will break.
// This is mainly a performance optimisation that avoids unnecessary memoization
// and using the costly `BestFitting` layout if it is already known that only the last variant
// can ever fit because the left breaks.
if last_target.inspect(f)?.will_break() {
if format_implicit_flat.is_none() && last_target_breaks {
return write!(
f,
[
@ -416,13 +549,29 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
);
}
let format_value = value.format().with_options(Parentheses::Never).memoized();
let format_value = format_with(|f| {
if let Some(format_implicit_flat) = format_implicit_flat.as_ref() {
if format_implicit_flat.string().is_fstring() {
// Remove any soft line breaks emitted by the f-string formatting.
// This is important when formatting f-strings as part of an assignment right side
// because `best_fit_parenthesize` will otherwise still try to break inner
// groups if wrapped in a `group(..).should_expand(true)`
let mut buffer = RemoveSoftLinesBuffer::new(&mut *f);
write!(buffer, [format_implicit_flat])
} else {
format_implicit_flat.fmt(f)
}
} else {
value.format().with_options(Parentheses::Never).fmt(f)
}
})
.memoized();
// Tries to fit the `left` and the `value` on a single line:
// ```python
// a = b = c
// ```
let format_flat = format_with(|f| {
let single_line = format_with(|f| {
write!(
f,
[
@ -443,19 +592,21 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
// c
// )
// ```
let format_parenthesize_value = format_with(|f| {
write!(
f,
[
last_target,
space(),
operator,
space(),
token("("),
block_indent(&format_args![format_value, inline_comments]),
token(")")
]
)
let flat_target_parenthesize_value = format_with(|f| {
write!(f, [last_target, space(), operator, space(), token("("),])?;
if is_join_implicit_concatenated_string_enabled(f.context()) {
group(&soft_block_indent(&format_args![
format_value,
inline_comments
]))
.should_expand(true)
.fmt(f)?;
} else {
block_indent(&format_args![format_value, inline_comments]).fmt(f)?;
}
token(")").fmt(f)
});
// Fall back to parenthesizing (or splitting) the last target part if we can't make the value
@ -466,17 +617,16 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
// "bbbbb"
// ] = c
// ```
let format_split_left = format_with(|f| {
let split_target_flat_value = format_with(|f| {
if is_join_implicit_concatenated_string_enabled(f.context()) {
group(&last_target).should_expand(true).fmt(f)?;
} else {
last_target.fmt(f)?;
}
write!(
f,
[
last_target,
space(),
operator,
space(),
format_value,
inline_comments
]
[space(), operator, space(), format_value, inline_comments]
)
});
@ -486,7 +636,7 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
// For attribute chains that contain any parenthesized value: Try expanding the parenthesized value first.
if value.is_call_expr() || value.is_subscript_expr() || value.is_attribute_expr() {
best_fitting![
format_flat,
single_line,
// Avoid parenthesizing the call expression if the `(` fit on the line
format_args![
last_target,
@ -495,12 +645,165 @@ impl Format<PyFormatContext<'_>> for FormatStatementsLastExpression<'_> {
space(),
group(&format_value).should_expand(true),
],
format_parenthesize_value,
format_split_left
flat_target_parenthesize_value,
split_target_flat_value
]
.fmt(f)
} else if let Some(format_implicit_flat) = &format_implicit_flat {
// F-String containing an expression with a magic trailing comma, a comment, or a
// multiline debug expression should never be joined. Use the default layout.
//
// ```python
// aaaa = f"abcd{[
// 1,
// 2,
// ]}" "more"
// ```
if format_implicit_flat.string().is_fstring()
&& format_value.inspect(f)?.will_break()
{
inline_comments.mark_unformatted();
return write!(
f,
[
before_operator,
space(),
operator,
space(),
maybe_parenthesize_expression(
value,
*statement,
Parenthesize::IfBreaks
)
]
);
}
let group_id = f.group_id("optional_parentheses");
let format_expanded = format_with(|f| {
let f = &mut WithNodeLevel::new(NodeLevel::Expression(Some(group_id)), f);
FormatImplicitConcatenatedStringExpanded::new(
StringLike::try_from(*value).unwrap(),
)
.fmt(f)
})
.memoized();
// Keep the target flat, parenthesize the value, and keep it multiline.
//
// ```python
// Literal[ "a", "b"] = (
// "looooooooooooooooooooooooooooooong"
// "string"
// ) # comment
// ```
let flat_target_value_parenthesized_multiline = format_with(|f| {
write!(
f,
[
last_target,
space(),
operator,
space(),
token("("),
group(&soft_block_indent(&format_expanded))
.with_group_id(Some(group_id))
.should_expand(true),
token(")"),
inline_comments
]
)
});
// Expand the parent and parenthesize the joined string with the inlined comment.
//
// ```python
// Literal[
// "a",
// "b",
// ] = (
// "not that long string" # comment
// )
// ```
let split_target_value_parenthesized_flat = format_with(|f| {
write!(
f,
[
group(&last_target).should_expand(true),
space(),
operator,
space(),
token("("),
group(&soft_block_indent(&format_args![
format_value,
inline_comments
]))
.should_expand(true),
token(")")
]
)
});
// The most expanded variant: Expand both the target and the string.
//
// ```python
// Literal[
// "a",
// "b",
// ] = (
// "looooooooooooooooooooooooooooooong"
// "string"
// ) # comment
// ```
let split_target_value_parenthesized_multiline = format_with(|f| {
write!(
f,
[
group(&last_target).should_expand(true),
space(),
operator,
space(),
token("("),
group(&soft_block_indent(&format_expanded))
.with_group_id(Some(group_id))
.should_expand(true),
token(")"),
inline_comments
]
)
});
// This is only a perf optimisation. No point in trying all the "flat-target"
// variants if we know that the last target must break.
if last_target_breaks {
best_fitting![
split_target_flat_value,
split_target_value_parenthesized_flat,
split_target_value_parenthesized_multiline,
]
.with_mode(BestFittingMode::AllLines)
.fmt(f)
} else {
best_fitting![
single_line,
flat_target_parenthesize_value,
flat_target_value_parenthesized_multiline,
split_target_flat_value,
split_target_value_parenthesized_flat,
split_target_value_parenthesized_multiline,
]
.with_mode(BestFittingMode::AllLines)
.fmt(f)
}
} else {
best_fitting![format_flat, format_parenthesize_value, format_split_left].fmt(f)
best_fitting![
single_line,
flat_target_parenthesize_value,
split_target_flat_value
]
.fmt(f)
}
}
}
@ -556,6 +859,12 @@ impl<'a> OptionalParenthesesInlinedComments<'a> {
comment.mark_formatted();
}
}
fn mark_unformatted(&self) {
for comment in self.expression {
comment.mark_unformatted();
}
}
}
impl Format<PyFormatContext<'_>> for OptionalParenthesesInlinedComments<'_> {

34
crates/ruff_python_formatter/src/statement/suite.rs
View file

@ -138,7 +138,7 @@ impl FormatRule<Suite, PyFormatContext<'_>> for FormatSuite {
SuiteKind::Function | SuiteKind::Class | SuiteKind::TopLevel => {
if let Some(docstring) =
DocstringStmt::try_from_statement(first, self.kind, source_type)
DocstringStmt::try_from_statement(first, self.kind, f.context())
{
SuiteChildStatement::Docstring(docstring)
} else {
@ -179,7 +179,7 @@ impl FormatRule<Suite, PyFormatContext<'_>> for FormatSuite {
// Insert a newline after a module level docstring, but treat
// it as a docstring otherwise. See: https://github.com/psf/black/pull/3932.
self.kind == SuiteKind::TopLevel
&& DocstringStmt::try_from_statement(first.statement(), self.kind, source_type)
&& DocstringStmt::try_from_statement(first.statement(), self.kind, f.context())
.is_some()
};
@ -785,37 +785,23 @@ impl<'a> DocstringStmt<'a> {
fn try_from_statement(
stmt: &'a Stmt,
suite_kind: SuiteKind,
source_type: PySourceType,
context: &PyFormatContext,
) -> Option<DocstringStmt<'a>> {
// Notebooks don't have a concept of modules, therefore, don't recognise the first string as the module docstring.
if source_type.is_ipynb() && suite_kind == SuiteKind::TopLevel {
if context.options().source_type().is_ipynb() && suite_kind == SuiteKind::TopLevel {
return None;
}
let Stmt::Expr(ast::StmtExpr { value, .. }) = stmt else {
return None;
};
match value.as_ref() {
Expr::StringLiteral(ast::ExprStringLiteral { value, .. })
if !value.is_implicit_concatenated() =>
{
Some(DocstringStmt {
docstring: stmt,
suite_kind,
})
}
_ => None,
}
Self::is_docstring_statement(stmt.as_expr_stmt()?, context).then_some(DocstringStmt {
docstring: stmt,
suite_kind,
})
}
pub(crate) fn is_docstring_statement(stmt: &StmtExpr, source_type: PySourceType) -> bool {
if source_type.is_ipynb() {
return false;
}
pub(crate) fn is_docstring_statement(stmt: &StmtExpr, context: &PyFormatContext) -> bool {
if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = stmt.value.as_ref() {
!value.is_implicit_concatenated()
|| !value.iter().any(|literal| context.comments().has(literal))
} else {
false
}

26
crates/ruff_python_formatter/src/string/docstring.rs
View file

@ -9,7 +9,7 @@ use std::{borrow::Cow, collections::VecDeque};
use regex::Regex;
use ruff_formatter::printer::SourceMapGeneration;
use ruff_python_ast::{str::Quote, StringFlags};
use ruff_python_ast::{str::Quote, AnyStringFlags, StringFlags};
use ruff_python_trivia::CommentRanges;
use {
ruff_formatter::{write, FormatOptions, IndentStyle, LineWidth, Printed},
@ -19,7 +19,10 @@ use {
};
use super::NormalizedString;
use crate::preview::is_docstring_code_block_in_docstring_indent_enabled;
use crate::preview::{
is_docstring_code_block_in_docstring_indent_enabled,
is_join_implicit_concatenated_string_enabled,
};
use crate::string::StringQuotes;
use crate::{prelude::*, DocstringCodeLineWidth, FormatModuleError};
@ -167,7 +170,7 @@ pub(crate) fn format(normalized: &NormalizedString, f: &mut PyFormatter) -> Form
if docstring[first.len()..].trim().is_empty() {
// For `"""\n"""` or other whitespace between the quotes, black keeps a single whitespace,
// but `""""""` doesn't get one inserted.
if needs_chaperone_space(normalized, trim_end)
if needs_chaperone_space(normalized.flags(), trim_end, f.context())
|| (trim_end.is_empty() && !docstring.is_empty())
{
space().fmt(f)?;
@ -207,7 +210,7 @@ pub(crate) fn format(normalized: &NormalizedString, f: &mut PyFormatter) -> Form
let trim_end = docstring
.as_ref()
.trim_end_matches(|c: char| c.is_whitespace() && c != '\n');
if needs_chaperone_space(normalized, trim_end) {
if needs_chaperone_space(normalized.flags(), trim_end, f.context()) {
space().fmt(f)?;
}
@ -1604,9 +1607,18 @@ fn docstring_format_source(
/// If the last line of the docstring is `content" """` or `content\ """`, we need a chaperone space
/// that avoids `content""""` and `content\"""`. This does only applies to un-escaped backslashes,
/// so `content\\ """` doesn't need a space while `content\\\ """` does.
fn needs_chaperone_space(normalized: &NormalizedString, trim_end: &str) -> bool {
trim_end.ends_with(normalized.flags().quote_style().as_char())
|| trim_end.chars().rev().take_while(|c| *c == '\\').count() % 2 == 1
pub(super) fn needs_chaperone_space(
flags: AnyStringFlags,
trim_end: &str,
context: &PyFormatContext,
) -> bool {
if trim_end.chars().rev().take_while(|c| *c == '\\').count() % 2 == 1 {
true
} else if is_join_implicit_concatenated_string_enabled(context) {
flags.is_triple_quoted() && trim_end.ends_with(flags.quote_style().as_char())
} else {
trim_end.ends_with(flags.quote_style().as_char())
}
}
#[derive(Copy, Clone, Debug)]

406
crates/ruff_python_formatter/src/string/implicit.rs Normal file
View file

@ -0,0 +1,406 @@
use std::borrow::Cow;
use ruff_formatter::{format_args, write, FormatContext};
use ruff_python_ast::str::Quote;
use ruff_python_ast::str_prefix::{
AnyStringPrefix, ByteStringPrefix, FStringPrefix, StringLiteralPrefix,
};
use ruff_python_ast::{AnyStringFlags, FStringElement, StringFlags, StringLike, StringLikePart};
use ruff_text_size::{Ranged, TextRange};
use crate::comments::{leading_comments, trailing_comments};
use crate::expression::parentheses::in_parentheses_only_soft_line_break_or_space;
use crate::other::f_string::{FStringContext, FStringLayout, FormatFString};
use crate::other::f_string_element::FormatFStringExpressionElement;
use crate::other::string_literal::StringLiteralKind;
use crate::prelude::*;
use crate::preview::{
is_f_string_formatting_enabled, is_join_implicit_concatenated_string_enabled,
};
use crate::string::docstring::needs_chaperone_space;
use crate::string::normalize::{
is_fstring_with_quoted_debug_expression,
is_fstring_with_triple_quoted_literal_expression_containing_quotes, QuoteMetadata,
};
use crate::string::{normalize_string, StringLikeExtensions, StringNormalizer, StringQuotes};
/// Formats any implicitly concatenated string. This could be any valid combination
/// of string, bytes or f-string literals.
pub(crate) struct FormatImplicitConcatenatedString<'a> {
string: StringLike<'a>,
}
impl<'a> FormatImplicitConcatenatedString<'a> {
pub(crate) fn new(string: impl Into<StringLike<'a>>) -> Self {
Self {
string: string.into(),
}
}
}
impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedString<'_> {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let expanded = FormatImplicitConcatenatedStringExpanded::new(self.string);
// If the string can be joined, try joining the implicit concatenated string into a single string
// if it fits on the line. Otherwise, parenthesize the string parts and format each part on its
// own line.
if let Some(flat) = FormatImplicitConcatenatedStringFlat::new(self.string, f.context()) {
write!(
f,
[if_group_fits_on_line(&flat), if_group_breaks(&expanded)]
)
} else {
expanded.fmt(f)
}
}
}
/// Formats an implicit concatenated string where parts are separated by a space or line break.
pub(crate) struct FormatImplicitConcatenatedStringExpanded<'a> {
string: StringLike<'a>,
}
impl<'a> FormatImplicitConcatenatedStringExpanded<'a> {
pub(crate) fn new(string: StringLike<'a>) -> Self {
assert!(string.is_implicit_concatenated());
Self { string }
}
}
impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedStringExpanded<'_> {
fn fmt(&self, f: &mut Formatter<PyFormatContext<'_>>) -> FormatResult<()> {
let comments = f.context().comments().clone();
let quoting = self.string.quoting(&f.context().locator());
let join_implicit_concatenated_string_enabled =
is_join_implicit_concatenated_string_enabled(f.context());
let mut joiner = f.join_with(in_parentheses_only_soft_line_break_or_space());
for part in self.string.parts() {
let format_part = format_with(|f: &mut PyFormatter| match part {
StringLikePart::String(part) => {
let kind = if self.string.is_fstring() {
#[allow(deprecated)]
StringLiteralKind::InImplicitlyConcatenatedFString(quoting)
} else {
StringLiteralKind::String
};
part.format().with_options(kind).fmt(f)
}
StringLikePart::Bytes(bytes_literal) => bytes_literal.format().fmt(f),
StringLikePart::FString(part) => FormatFString::new(part, quoting).fmt(f),
});
let part_comments = comments.leading_dangling_trailing(&part);
joiner.entry(&format_args![
(!join_implicit_concatenated_string_enabled).then_some(line_suffix_boundary()),
leading_comments(part_comments.leading),
format_part,
trailing_comments(part_comments.trailing)
]);
}
joiner.finish()
}
}
/// Formats an implicit concatenated string where parts are joined into a single string if possible.
pub(crate) struct FormatImplicitConcatenatedStringFlat<'a> {
string: StringLike<'a>,
flags: AnyStringFlags,
docstring: bool,
}
impl<'a> FormatImplicitConcatenatedStringFlat<'a> {
/// Creates a new formatter. Returns `None` if the string can't be merged into a single string.
pub(crate) fn new(string: StringLike<'a>, context: &PyFormatContext) -> Option<Self> {
fn merge_flags(string: StringLike, context: &PyFormatContext) -> Option<AnyStringFlags> {
if !is_join_implicit_concatenated_string_enabled(context) {
return None;
}
// Multiline strings can never fit on a single line.
if !string.is_fstring() && string.is_multiline(context.source()) {
return None;
}
let first_part = string.parts().next()?;
// The string is either a regular string, f-string, or bytes string.
let normalizer = StringNormalizer::from_context(context);
// Some if a part requires preserving its quotes.
let mut preserve_quotes_requirement: Option<Quote> = None;
// Early exit if it's known that this string can't be joined
for part in string.parts() {
// Similar to Black, don't collapse triple quoted and raw strings.
// We could technically join strings that are raw-strings and use the same quotes but lets not do this for now.
// Joining triple quoted strings is more complicated because an
// implicit concatenated string could become a docstring (if it's the first string in a block).
// That means the joined string formatting would have to call into
// the docstring formatting or otherwise guarantee that the output
// won't change on a second run.
if part.flags().is_triple_quoted() || part.flags().is_raw_string() {
return None;
}
// For now, preserve comments documenting a specific part over possibly
// collapsing onto a single line. Collapsing could result in pragma comments
// now covering more code.
if context.comments().leading_trailing(&part).next().is_some() {
return None;
}
if let StringLikePart::FString(fstring) = part {
if fstring.elements.iter().any(|element| match element {
// Same as for other literals. Multiline literals can't fit on a single line.
FStringElement::Literal(literal) => context
.locator()
.slice(literal.range())
.contains(['\n', '\r']),
FStringElement::Expression(expression) => {
if is_f_string_formatting_enabled(context) {
// Expressions containing comments can't be joined.
context.comments().contains_comments(expression.into())
} else {
// Multiline f-string expressions can't be joined if the f-string formatting is disabled because
// the string gets inserted in verbatim preserving the newlines.
context.locator().slice(expression).contains(['\n', '\r'])
}
}
}) {
return None;
}
// Avoid invalid syntax for pre Python 312:
// * When joining parts that have debug expressions with quotes: `f"{10 + len('bar')=}" f'{10 + len("bar")=}'
// * When joining parts that contain triple quoted strings with quotes: `f"{'''test ' '''}" f'{"""other " """}'`
if !context.options().target_version().supports_pep_701() {
if is_fstring_with_quoted_debug_expression(fstring, context)
|| is_fstring_with_triple_quoted_literal_expression_containing_quotes(
fstring, context,
)
{
if preserve_quotes_requirement
.is_some_and(|quote| quote != part.flags().quote_style())
{
return None;
}
preserve_quotes_requirement = Some(part.flags().quote_style());
}
}
}
}
// The string is either a regular string, f-string, or bytes string.
let mut merged_quotes: Option<QuoteMetadata> = None;
// Only preserve the string type but disregard the `u` and `r` prefixes.
// * It's not necessary to preserve the `r` prefix because Ruff doesn't support joining raw strings (we shouldn't get here).
// * It's not necessary to preserve the `u` prefix because Ruff discards the `u` prefix (it's meaningless in Python 3+)
let prefix = match string {
StringLike::String(_) => AnyStringPrefix::Regular(StringLiteralPrefix::Empty),
StringLike::Bytes(_) => AnyStringPrefix::Bytes(ByteStringPrefix::Regular),
StringLike::FString(_) => AnyStringPrefix::Format(FStringPrefix::Regular),
};
// Only determining the preferred quote for the first string is sufficient
// because we don't support joining triple quoted strings with non triple quoted strings.
let quote = if let Ok(preferred_quote) =
Quote::try_from(normalizer.preferred_quote_style(first_part))
{
for part in string.parts() {
let part_quote_metadata =
QuoteMetadata::from_part(part, context, preferred_quote);
if let Some(merged) = merged_quotes.as_mut() {
*merged = part_quote_metadata.merge(merged)?;
} else {
merged_quotes = Some(part_quote_metadata);
}
}
merged_quotes?.choose(preferred_quote)
} else {
// Use the quotes of the first part if the quotes should be preserved.
first_part.flags().quote_style()
};
Some(AnyStringFlags::new(prefix, quote, false))
}
if !string.is_implicit_concatenated() {
return None;
}
Some(Self {
flags: merge_flags(string, context)?,
string,
docstring: false,
})
}
pub(crate) fn set_docstring(&mut self, is_docstring: bool) {
self.docstring = is_docstring;
}
pub(crate) fn string(&self) -> StringLike<'a> {
self.string
}
}
impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedStringFlat<'_> {
fn fmt(&self, f: &mut Formatter<PyFormatContext<'_>>) -> FormatResult<()> {
// Merges all string parts into a single string.
let quotes = StringQuotes::from(self.flags);
write!(f, [self.flags.prefix(), quotes])?;
let mut parts = self.string.parts().peekable();
// Trim implicit concatenated strings in docstring positions.
// Skip over any trailing parts that are all whitespace.
// Leading parts are handled as part of the formatting loop below.
if self.docstring {
for part in self.string.parts().rev() {
assert!(part.is_string_literal());
if f.context()
.locator()
.slice(part.content_range())
.trim()
.is_empty()
{
// Don't format the part.
parts.next_back();
} else {
break;
}
}
}
let mut first_non_empty = self.docstring;
while let Some(part) = parts.next() {
match part {
StringLikePart::String(_) | StringLikePart::Bytes(_) => {
FormatLiteralContent {
range: part.content_range(),
flags: self.flags,
is_fstring: false,
trim_start: first_non_empty && self.docstring,
trim_end: self.docstring && parts.peek().is_none(),
}
.fmt(f)?;
if first_non_empty {
first_non_empty = f
.context()
.locator()
.slice(part.content_range())
.trim_start()
.is_empty();
}
}
StringLikePart::FString(f_string) => {
if is_f_string_formatting_enabled(f.context()) {
for element in &f_string.elements {
match element {
FStringElement::Literal(literal) => {
FormatLiteralContent {
range: literal.range(),
flags: self.flags,
is_fstring: true,
trim_end: false,
trim_start: false,
}
.fmt(f)?;
}
// Formatting the expression here and in the expanded version is safe **only**
// because we assert that the f-string never contains any comments.
FStringElement::Expression(expression) => {
let context = FStringContext::new(
self.flags,
FStringLayout::from_f_string(
f_string,
&f.context().locator(),
),
);
FormatFStringExpressionElement::new(expression, context)
.fmt(f)?;
}
}
}
} else {
FormatLiteralContent {
range: part.content_range(),
flags: self.flags,
is_fstring: true,
trim_end: false,
trim_start: false,
}
.fmt(f)?;
}
}
}
}
quotes.fmt(f)
}
}
struct FormatLiteralContent {
range: TextRange,
flags: AnyStringFlags,
is_fstring: bool,
trim_start: bool,
trim_end: bool,
}
impl Format<PyFormatContext<'_>> for FormatLiteralContent {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let content = f.context().locator().slice(self.range);
let mut normalized = normalize_string(
content,
0,
self.flags,
self.flags.is_f_string() && !self.is_fstring,
true,
false,
);
// Trim the start and end of the string if it's the first or last part of a docstring.
// This is rare, so don't bother with optimizing to use `Cow`.
if self.trim_start {
let trimmed = normalized.trim_start();
if trimmed.len() < normalized.len() {
normalized = trimmed.to_string().into();
}
}
if self.trim_end {
let trimmed = normalized.trim_end();
if trimmed.len() < normalized.len() {
normalized = trimmed.to_string().into();
}
}
if !normalized.is_empty() {
match &normalized {
Cow::Borrowed(_) => source_text_slice(self.range).fmt(f)?,
Cow::Owned(normalized) => text(normalized).fmt(f)?,
}
if self.trim_end && needs_chaperone_space(self.flags, &normalized, f.context()) {
space().fmt(f)?;
}
}
Ok(())
}
}

67
crates/ruff_python_formatter/src/string/mod.rs
View file

@ -1,25 +1,21 @@
use memchr::memchr2;
pub(crate) use normalize::{normalize_string, NormalizedString, StringNormalizer};
use ruff_formatter::format_args;
use ruff_python_ast::str::Quote;
use ruff_python_ast::{
self as ast,
str_prefix::{AnyStringPrefix, StringLiteralPrefix},
AnyStringFlags, StringFlags, StringLike, StringLikePart,
AnyStringFlags, StringFlags,
};
use ruff_source_file::Locator;
use ruff_text_size::Ranged;
use crate::comments::{leading_comments, trailing_comments};
use crate::expression::expr_f_string::f_string_quoting;
use crate::expression::parentheses::in_parentheses_only_soft_line_break_or_space;
use crate::other::f_string::FormatFString;
use crate::other::string_literal::StringLiteralKind;
use crate::prelude::*;
use crate::QuoteStyle;
pub(crate) mod docstring;
pub(crate) mod implicit;
mod normalize;
#[derive(Copy, Clone, Debug, Default)]
@ -29,57 +25,6 @@ pub(crate) enum Quoting {
Preserve,
}
/// Formats any implicitly concatenated string. This could be any valid combination
/// of string, bytes or f-string literals.
pub(crate) struct FormatImplicitConcatenatedString<'a> {
string: StringLike<'a>,
}
impl<'a> FormatImplicitConcatenatedString<'a> {
pub(crate) fn new(string: impl Into<StringLike<'a>>) -> Self {
Self {
string: string.into(),
}
}
}
impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedString<'_> {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let comments = f.context().comments().clone();
let quoting = self.string.quoting(&f.context().locator());
let mut joiner = f.join_with(in_parentheses_only_soft_line_break_or_space());
for part in self.string.parts() {
let part_comments = comments.leading_dangling_trailing(&part);
let format_part = format_with(|f: &mut PyFormatter| match part {
StringLikePart::String(part) => {
let kind = if self.string.is_fstring() {
#[allow(deprecated)]
StringLiteralKind::InImplicitlyConcatenatedFString(quoting)
} else {
StringLiteralKind::String
};
part.format().with_options(kind).fmt(f)
}
StringLikePart::Bytes(bytes_literal) => bytes_literal.format().fmt(f),
StringLikePart::FString(part) => FormatFString::new(part, quoting).fmt(f),
});
joiner.entry(&format_args![
line_suffix_boundary(),
leading_comments(part_comments.leading),
format_part,
trailing_comments(part_comments.trailing)
]);
}
joiner.finish()
}
}
impl Format<PyFormatContext<'_>> for AnyStringPrefix {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
// Remove the unicode prefix `u` if any because it is meaningless in Python 3+.
@ -159,12 +104,10 @@ impl StringLikeExtensions for ast::StringLike<'_> {
fn is_multiline(&self, source: &str) -> bool {
match self {
Self::String(_) | Self::Bytes(_) => {
self.parts()
.next()
.is_some_and(|part| part.flags().is_triple_quoted())
Self::String(_) | Self::Bytes(_) => self.parts().any(|part| {
part.flags().is_triple_quoted()
&& memchr2(b'\n', b'\r', source[self.range()].as_bytes()).is_some()
}
}),
Self::FString(fstring) => {
memchr2(b'\n', b'\r', source[fstring.range].as_bytes()).is_some()
}

68
crates/ruff_python_formatter/src/string/normalize.rs
View file

@ -44,6 +44,9 @@ impl<'a, 'src> StringNormalizer<'a, 'src> {
/// The formatter should use the preferred quote style unless
/// it can't because the string contains the preferred quotes OR
/// it leads to more escaping.
///
/// Note: If you add more cases here where we return `QuoteStyle::Preserve`,
/// make sure to also add them to [`FormatImplicitConcatenatedStringFlat::new`].
pub(super) fn preferred_quote_style(&self, string: StringLikePart) -> QuoteStyle {
match self.quoting {
Quoting::Preserve => QuoteStyle::Preserve,
@ -205,6 +208,8 @@ impl<'a, 'src> StringNormalizer<'a, 'src> {
quote_selection.flags,
// TODO: Remove the `b'{'` in `choose_quotes` when promoting the
// `format_fstring` preview style
false,
false,
is_f_string_formatting_enabled(self.context),
)
} else {
@ -598,6 +603,8 @@ pub(crate) fn normalize_string(
input: &str,
start_offset: usize,
new_flags: AnyStringFlags,
escape_braces: bool,
flip_nested_fstring_quotes: bool,
format_f_string: bool,
) -> Cow<str> {
// The normalized string if `input` is not yet normalized.
@ -620,16 +627,24 @@ pub(crate) fn normalize_string(
while let Some((index, c)) = chars.next() {
if matches!(c, '{' | '}') && is_fstring {
if chars.peek().copied().is_some_and(|(_, next)| next == c) {
// Skip over the second character of the double braces
chars.next();
} else if c == '{' {
formatted_value_nesting += 1;
} else {
// Safe to assume that `c == '}'` here because of the matched pattern above
formatted_value_nesting = formatted_value_nesting.saturating_sub(1);
if escape_braces {
// Escape `{` and `}` when converting a regular string literal to an f-string literal.
output.push_str(&input[last_index..=index]);
output.push(c);
last_index = index + c.len_utf8();
continue;
} else if is_fstring {
if chars.peek().copied().is_some_and(|(_, next)| next == c) {
// Skip over the second character of the double braces
chars.next();
} else if c == '{' {
formatted_value_nesting += 1;
} else {
// Safe to assume that `c == '}'` here because of the matched pattern above
formatted_value_nesting = formatted_value_nesting.saturating_sub(1);
}
continue;
}
continue;
}
if c == '\r' {
@ -697,6 +712,14 @@ pub(crate) fn normalize_string(
output.push('\\');
output.push(c);
last_index = index + preferred_quote.len_utf8();
} else if c == preferred_quote
&& flip_nested_fstring_quotes
&& formatted_value_nesting > 0
{
// Flip the quotes
output.push_str(&input[last_index..index]);
output.push(opposite_quote);
last_index = index + preferred_quote.len_utf8();
}
}
}
@ -981,6 +1004,7 @@ pub(super) fn is_fstring_with_triple_quoted_literal_expression_containing_quotes
mod tests {
use std::borrow::Cow;
use ruff_python_ast::str_prefix::FStringPrefix;
use ruff_python_ast::{
str::Quote,
str_prefix::{AnyStringPrefix, ByteStringPrefix},
@ -1013,9 +1037,35 @@ mod tests {
Quote::Double,
false,
),
false,
false,
true,
);
assert_eq!(r"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a", &normalized);
}
#[test]
fn normalize_nested_fstring() {
let input =
r#"With single quote: ' {my_dict['foo']} With double quote: " {my_dict["bar"]}"#;
let normalized = normalize_string(
input,
0,
AnyStringFlags::new(
AnyStringPrefix::Format(FStringPrefix::Regular),
Quote::Double,
false,
),
false,
true,
false,
);
assert_eq!(
"With single quote: ' {my_dict['foo']} With double quote: \\\" {my_dict['bar']}",
&normalized
);
}
}