mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-17 09:00:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Extract LineIndex independent methods from Locator (#13938)
Some checks are pending
CI / Fuzz the parser (push) Blocked by required conditions
Details
CI / test scripts (push) Blocked by required conditions
Details
CI / ecosystem (push) Blocked by required conditions
Details
CI / cargo shear (push) Blocked by required conditions
Details
CI / Determine changes (push) Waiting to run
Details
CI / cargo fmt (push) Waiting to run
Details
CI / cargo clippy (push) Blocked by required conditions
Details
CI / python package (push) Waiting to run
Details
CI / cargo test (linux) (push) Blocked by required conditions
Details
CI / cargo test (windows) (push) Blocked by required conditions
Details
CI / cargo test (wasm) (push) Blocked by required conditions
Details
CI / cargo build (release) (push) Blocked by required conditions
Details
CI / cargo build (msrv) (push) Blocked by required conditions
Details
CI / cargo fuzz (push) Blocked by required conditions
Details
CI / pre-commit (push) Waiting to run
Details
CI / mkdocs (push) Waiting to run
Details
CI / formatter instabilities and black similarity (push) Blocked by required conditions
Details
CI / test ruff-lsp (push) Blocked by required conditions
Details
CI / benchmarks (push) Blocked by required conditions
Details

Browse source
This commit is contained in:
Micha Reiser 2024-10-28 08:53:41 +01:00 • committed by GitHub
parent f8eb547fb4
commit 9f3a38d408
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
171 changed files with 1348 additions and 1284 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/ruff_python_formatter/src/comments/mod.rs
View file

@ -98,7 +98,6 @@ pub(crate) use format::{
use ruff_formatter::{SourceCode, SourceCodeSlice};
use ruff_python_ast::AnyNodeRef;
use ruff_python_trivia::{CommentLinePosition, CommentRanges, SuppressionKind};
use ruff_source_file::Locator;
use ruff_text_size::{Ranged, TextRange};
pub(crate) use visitor::collect_comments;
@ -258,8 +257,7 @@ impl<'a> Comments<'a> {
let map = if comment_ranges.is_empty() {
CommentsMap::new()
} else {
let mut builder =
CommentsMapBuilder::new(Locator::new(source_code.as_str()), comment_ranges);
let mut builder = CommentsMapBuilder::new(source_code.as_str(), comment_ranges);
CommentsVisitor::new(source_code, comment_ranges, &mut builder).visit(root);
builder.finish()
};

271
crates/ruff_python_formatter/src/comments/placement.rs
View file

@ -9,7 +9,7 @@ use ruff_python_trivia::{
find_only_token_in_range, first_non_trivia_token, indentation_at_offset, BackwardsTokenizer,
CommentRanges, SimpleToken, SimpleTokenKind, SimpleTokenizer,
};
use ruff_source_file::Locator;
use ruff_source_file::LineRanges;
use ruff_text_size::{Ranged, TextLen, TextRange};
use crate::comments::visitor::{CommentPlacement, DecoratedComment};
@ -23,12 +23,12 @@ use crate::pattern::pattern_match_sequence::SequenceType;
pub(super) fn place_comment<'a>(
comment: DecoratedComment<'a>,
comment_ranges: &CommentRanges,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
handle_parenthesized_comment(comment, locator)
.or_else(|comment| handle_end_of_line_comment_around_body(comment, locator))
.or_else(|comment| handle_own_line_comment_around_body(comment, locator))
.or_else(|comment| handle_enclosed_comment(comment, comment_ranges, locator))
handle_parenthesized_comment(comment, source)
.or_else(|comment| handle_end_of_line_comment_around_body(comment, source))
.or_else(|comment| handle_own_line_comment_around_body(comment, source))
.or_else(|comment| handle_enclosed_comment(comment, comment_ranges, source))
}
/// Handle parenthesized comments. A parenthesized comment is a comment that appears within a
@ -71,7 +71,7 @@ pub(super) fn place_comment<'a>(
/// comment is a leading comment of the following node.
fn handle_parenthesized_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
// As a special-case, ignore comments within f-strings, like:
// ```python
@ -133,7 +133,7 @@ fn handle_parenthesized_comment<'a>(
// ]
// ```
let range = TextRange::new(preceding.end(), comment.start());
let tokenizer = SimpleTokenizer::new(locator.contents(), range);
let tokenizer = SimpleTokenizer::new(source, range);
if tokenizer
.skip_trivia()
.take_while(|token| {
@ -146,7 +146,7 @@ fn handle_parenthesized_comment<'a>(
debug_assert!(
!matches!(token.kind, SimpleTokenKind::Bogus),
"Unexpected token between nodes: `{:?}`",
locator.slice(range)
&source[range]
);
token.kind() == SimpleTokenKind::LParen
})
@ -164,7 +164,7 @@ fn handle_parenthesized_comment<'a>(
// ]
// ```
let range = TextRange::new(comment.end(), following.start());
let tokenizer = SimpleTokenizer::new(locator.contents(), range);
let tokenizer = SimpleTokenizer::new(source, range);
if tokenizer
.skip_trivia()
.take_while(|token| {
@ -177,7 +177,7 @@ fn handle_parenthesized_comment<'a>(
debug_assert!(
!matches!(token.kind, SimpleTokenKind::Bogus),
"Unexpected token between nodes: `{:?}`",
locator.slice(range)
&source[range]
);
token.kind() == SimpleTokenKind::RParen
})
@ -192,61 +192,61 @@ fn handle_parenthesized_comment<'a>(
fn handle_enclosed_comment<'a>(
comment: DecoratedComment<'a>,
comment_ranges: &CommentRanges,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
match comment.enclosing_node() {
AnyNodeRef::Parameters(parameters) => {
handle_parameters_separator_comment(comment, parameters, locator).or_else(|comment| {
if are_parameters_parenthesized(parameters, locator.contents()) {
handle_bracketed_end_of_line_comment(comment, locator)
handle_parameters_separator_comment(comment, parameters, source).or_else(|comment| {
if are_parameters_parenthesized(parameters, source) {
handle_bracketed_end_of_line_comment(comment, source)
} else {
CommentPlacement::Default(comment)
}
})
}
AnyNodeRef::Parameter(parameter) => handle_parameter_comment(comment, parameter, locator),
AnyNodeRef::Parameter(parameter) => handle_parameter_comment(comment, parameter, source),
AnyNodeRef::Arguments(_) | AnyNodeRef::TypeParams(_) | AnyNodeRef::PatternArguments(_) => {
handle_bracketed_end_of_line_comment(comment, locator)
handle_bracketed_end_of_line_comment(comment, source)
}
AnyNodeRef::Comprehension(comprehension) => {
handle_comprehension_comment(comment, comprehension, locator)
handle_comprehension_comment(comment, comprehension, source)
}
AnyNodeRef::ExprAttribute(attribute) => {
handle_attribute_comment(comment, attribute, locator)
handle_attribute_comment(comment, attribute, source)
}
AnyNodeRef::ExprBinOp(binary_expression) => {
handle_trailing_binary_expression_left_or_operator_comment(
comment,
binary_expression,
locator,
source,
)
}
AnyNodeRef::ExprBoolOp(_) | AnyNodeRef::ExprCompare(_) => {
handle_trailing_binary_like_comment(comment, locator)
handle_trailing_binary_like_comment(comment, source)
}
AnyNodeRef::Keyword(keyword) => handle_keyword_comment(comment, keyword, locator),
AnyNodeRef::Keyword(keyword) => handle_keyword_comment(comment, keyword, source),
AnyNodeRef::PatternKeyword(pattern_keyword) => {
handle_pattern_keyword_comment(comment, pattern_keyword, locator)
handle_pattern_keyword_comment(comment, pattern_keyword, source)
}
AnyNodeRef::ExprUnaryOp(unary_op) => handle_unary_op_comment(comment, unary_op, locator),
AnyNodeRef::ExprNamed(_) => handle_named_expr_comment(comment, locator),
AnyNodeRef::ExprLambda(lambda) => handle_lambda_comment(comment, lambda, locator),
AnyNodeRef::ExprDict(_) => handle_dict_unpacking_comment(comment, locator)
.or_else(|comment| handle_bracketed_end_of_line_comment(comment, locator))
.or_else(|comment| handle_key_value_comment(comment, locator)),
AnyNodeRef::ExprDictComp(_) => handle_key_value_comment(comment, locator)
.or_else(|comment| handle_bracketed_end_of_line_comment(comment, locator)),
AnyNodeRef::ExprIf(expr_if) => handle_expr_if_comment(comment, expr_if, locator),
AnyNodeRef::ExprUnaryOp(unary_op) => handle_unary_op_comment(comment, unary_op, source),
AnyNodeRef::ExprNamed(_) => handle_named_expr_comment(comment, source),
AnyNodeRef::ExprLambda(lambda) => handle_lambda_comment(comment, lambda, source),
AnyNodeRef::ExprDict(_) => handle_dict_unpacking_comment(comment, source)
.or_else(|comment| handle_bracketed_end_of_line_comment(comment, source))
.or_else(|comment| handle_key_value_comment(comment, source)),
AnyNodeRef::ExprDictComp(_) => handle_key_value_comment(comment, source)
.or_else(|comment| handle_bracketed_end_of_line_comment(comment, source)),
AnyNodeRef::ExprIf(expr_if) => handle_expr_if_comment(comment, expr_if, source),
AnyNodeRef::ExprSlice(expr_slice) => {
handle_slice_comments(comment, expr_slice, comment_ranges, locator)
handle_slice_comments(comment, expr_slice, comment_ranges, source)
}
AnyNodeRef::ExprStarred(starred) => {
handle_trailing_expression_starred_star_end_of_line_comment(comment, starred, locator)
handle_trailing_expression_starred_star_end_of_line_comment(comment, starred, source)
}
AnyNodeRef::ExprSubscript(expr_subscript) => {
if let Expr::Slice(expr_slice) = expr_subscript.slice.as_ref() {
return handle_slice_comments(comment, expr_slice, comment_ranges, locator);
return handle_slice_comments(comment, expr_slice, comment_ranges, source);
}
// Handle non-slice subscript end-of-line comments coming after the `[`
@ -262,7 +262,7 @@ fn handle_enclosed_comment<'a>(
{
// Ensure that there are no tokens between the open bracket and the comment.
let mut lexer = SimpleTokenizer::new(
locator.contents(),
source,
TextRange::new(expr_subscript.value.end(), comment.start()),
)
.skip_trivia();
@ -288,26 +288,24 @@ fn handle_enclosed_comment<'a>(
AnyNodeRef::ModModule(module) => {
handle_trailing_module_comment(module, comment).or_else(|comment| {
handle_module_level_own_line_comment_before_class_or_function_comment(
comment, locator,
comment, source,
)
})
}
AnyNodeRef::WithItem(_) => handle_with_item_comment(comment, locator),
AnyNodeRef::WithItem(_) => handle_with_item_comment(comment, source),
AnyNodeRef::PatternMatchSequence(pattern_match_sequence) => {
if SequenceType::from_pattern(pattern_match_sequence, locator.contents())
.is_parenthesized()
{
handle_bracketed_end_of_line_comment(comment, locator)
if SequenceType::from_pattern(pattern_match_sequence, source).is_parenthesized() {
handle_bracketed_end_of_line_comment(comment, source)
} else {
CommentPlacement::Default(comment)
}
}
AnyNodeRef::PatternMatchClass(class) => handle_pattern_match_class_comment(comment, class),
AnyNodeRef::PatternMatchAs(_) => handle_pattern_match_as_comment(comment, locator),
AnyNodeRef::PatternMatchAs(_) => handle_pattern_match_as_comment(comment, source),
AnyNodeRef::PatternMatchStar(_) => handle_pattern_match_star_comment(comment),
AnyNodeRef::PatternMatchMapping(pattern) => {
handle_bracketed_end_of_line_comment(comment, locator)
.or_else(|comment| handle_pattern_match_mapping_comment(comment, pattern, locator))
handle_bracketed_end_of_line_comment(comment, source)
.or_else(|comment| handle_pattern_match_mapping_comment(comment, pattern, source))
}
AnyNodeRef::StmtFunctionDef(_) => handle_leading_function_with_decorators_comment(comment),
AnyNodeRef::StmtClassDef(class_def) => {
@ -343,19 +341,19 @@ fn handle_enclosed_comment<'a>(
) {
CommentPlacement::trailing(comment.enclosing_node(), comment)
} else {
handle_bracketed_end_of_line_comment(comment, locator)
handle_bracketed_end_of_line_comment(comment, source)
}
}
AnyNodeRef::ExprList(_)
| AnyNodeRef::ExprSet(_)
| AnyNodeRef::ExprListComp(_)
| AnyNodeRef::ExprSetComp(_) => handle_bracketed_end_of_line_comment(comment, locator),
| AnyNodeRef::ExprSetComp(_) => handle_bracketed_end_of_line_comment(comment, source),
AnyNodeRef::ExprTuple(ast::ExprTuple {
parenthesized: true,
..
}) => handle_bracketed_end_of_line_comment(comment, locator),
}) => handle_bracketed_end_of_line_comment(comment, source),
AnyNodeRef::ExprGenerator(generator) if generator.parenthesized => {
handle_bracketed_end_of_line_comment(comment, locator)
handle_bracketed_end_of_line_comment(comment, source)
}
_ => CommentPlacement::Default(comment),
}
@ -364,7 +362,7 @@ fn handle_enclosed_comment<'a>(
/// Handle an end-of-line comment around a body.
fn handle_end_of_line_comment_around_body<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if comment.line_position().is_own_line() {
return CommentPlacement::Default(comment);
@ -379,13 +377,10 @@ fn handle_end_of_line_comment_around_body<'a>(
// ```
if let Some(following) = comment.following_node() {
if following.is_first_statement_in_body(comment.enclosing_node())
&& SimpleTokenizer::new(
locator.contents(),
TextRange::new(comment.end(), following.start()),
)
.skip_trivia()
.next()
.is_none()
&& SimpleTokenizer::new(source, TextRange::new(comment.end(), following.start()))
.skip_trivia()
.next()
.is_none()
{
return CommentPlacement::dangling(comment.enclosing_node(), comment);
}
@ -436,7 +431,7 @@ fn handle_end_of_line_comment_around_body<'a>(
/// ```
fn handle_own_line_comment_around_body<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if comment.line_position().is_end_of_line() {
return CommentPlacement::Default(comment);
@ -458,24 +453,22 @@ fn handle_own_line_comment_around_body<'a>(
// # default placement comment
// def inline_after_else(): ...
// ```
let maybe_token = SimpleTokenizer::new(
locator.contents(),
TextRange::new(preceding.end(), comment.start()),
)
.skip_trivia()
.next();
let maybe_token =
SimpleTokenizer::new(source, TextRange::new(preceding.end(), comment.start()))
.skip_trivia()
.next();
if maybe_token.is_some() {
return CommentPlacement::Default(comment);
}
// Check if we're between bodies and should attach to the following body.
handle_own_line_comment_between_branches(comment, preceding, locator)
handle_own_line_comment_between_branches(comment, preceding, source)
.or_else(|comment| {
// Otherwise, there's no following branch or the indentation is too deep, so attach to the
// recursively last statement in the preceding body with the matching indentation.
handle_own_line_comment_after_branch(comment, preceding, locator)
handle_own_line_comment_after_branch(comment, preceding, source)
})
.or_else(|comment| handle_own_line_comment_between_statements(comment, locator))
.or_else(|comment| handle_own_line_comment_between_statements(comment, source))
}
/// Handles own-line comments between statements. If an own-line comment is between two statements,
@ -500,7 +493,7 @@ fn handle_own_line_comment_around_body<'a>(
/// ```
fn handle_own_line_comment_between_statements<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let Some(preceding) = comment.preceding_node() else {
return CommentPlacement::Default(comment);
@ -540,7 +533,7 @@ fn handle_own_line_comment_between_statements<'a>(
//
// y = 2
// ```
if max_empty_lines(locator.slice(TextRange::new(comment.end(), following.start()))) == 0 {
if max_empty_lines(&source[TextRange::new(comment.end(), following.start())]) == 0 {
CommentPlacement::leading(following, comment)
} else {
CommentPlacement::trailing(preceding, comment)
@ -559,7 +552,7 @@ fn handle_own_line_comment_between_statements<'a>(
fn handle_own_line_comment_between_branches<'a>(
comment: DecoratedComment<'a>,
preceding: AnyNodeRef<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
// The following statement must be the first statement in an alternate body, otherwise check
// if it's a comment after the final body and handle that case
@ -572,9 +565,9 @@ fn handle_own_line_comment_between_branches<'a>(
// It depends on the indentation level of the comment if it is a leading comment for the
// following branch or if it a trailing comment of the previous body's last statement.
let comment_indentation = comment_indentation_after(preceding, comment.range(), locator);
let comment_indentation = comment_indentation_after(preceding, comment.range(), source);
let preceding_indentation = indentation(locator, &preceding)
let preceding_indentation = indentation(source, &preceding)
.unwrap_or_default()
.text_len();
@ -648,7 +641,7 @@ fn handle_own_line_comment_between_branches<'a>(
fn handle_own_line_comment_after_branch<'a>(
comment: DecoratedComment<'a>,
preceding: AnyNodeRef<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let Some(last_child) = preceding.last_child_in_body() else {
return CommentPlacement::Default(comment);
@ -656,7 +649,7 @@ fn handle_own_line_comment_after_branch<'a>(
// We only care about the length because indentations with mixed spaces and tabs are only valid if
// the indent-level doesn't depend on the tab width (the indent level must be the same if the tab width is 1 or 8).
let comment_indentation = comment_indentation_after(preceding, comment.range(), locator);
let comment_indentation = comment_indentation_after(preceding, comment.range(), source);
// Keep the comment on the entire statement in case it's a trailing comment
// ```python
@ -667,7 +660,7 @@ fn handle_own_line_comment_after_branch<'a>(
// # Trailing if comment
// ```
// Here we keep the comment a trailing comment of the `if`
let preceding_indentation = indentation_at_offset(preceding.start(), locator)
let preceding_indentation = indentation_at_offset(preceding.start(), source)
.unwrap_or_default()
.text_len();
if comment_indentation == preceding_indentation {
@ -678,7 +671,7 @@ fn handle_own_line_comment_after_branch<'a>(
let mut last_child_in_parent = last_child;
loop {
let child_indentation = indentation(locator, &last_child_in_parent)
let child_indentation = indentation(source, &last_child_in_parent)
.unwrap_or_default()
.text_len();
@ -739,9 +732,9 @@ fn handle_own_line_comment_after_branch<'a>(
fn handle_parameters_separator_comment<'a>(
comment: DecoratedComment<'a>,
parameters: &Parameters,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let (slash, star) = find_parameter_separators(locator.contents(), parameters);
let (slash, star) = find_parameter_separators(source, parameters);
let placement = assign_argument_separator_comment_placement(
slash.as_ref(),
star.as_ref(),
@ -768,10 +761,10 @@ fn handle_parameters_separator_comment<'a>(
fn handle_parameter_comment<'a>(
comment: DecoratedComment<'a>,
parameter: &'a Parameter,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if parameter.annotation.as_deref().is_some() {
let colon = first_non_trivia_token(parameter.name.end(), locator.contents()).expect(
let colon = first_non_trivia_token(parameter.name.end(), source).expect(
"A annotated parameter should have a colon following its name when it is valid syntax.",
);
@ -804,7 +797,7 @@ fn handle_parameter_comment<'a>(
fn handle_trailing_binary_expression_left_or_operator_comment<'a>(
comment: DecoratedComment<'a>,
binary_expression: &'a ast::ExprBinOp,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
// Only if there's a preceding node (in which case, the preceding node is `left`).
if comment.preceding_node().is_none() || comment.following_node().is_none() {
@ -816,7 +809,7 @@ fn handle_trailing_binary_expression_left_or_operator_comment<'a>(
binary_expression.right.start(),
);
let mut tokens = SimpleTokenizer::new(locator.contents(), between_operands_range)
let mut tokens = SimpleTokenizer::new(source, between_operands_range)
.skip_trivia()
.skip_while(|token| token.kind == SimpleTokenKind::RParen);
let operator_offset = tokens
@ -836,10 +829,10 @@ fn handle_trailing_binary_expression_left_or_operator_comment<'a>(
CommentPlacement::trailing(binary_expression.left.as_ref(), comment)
} else if comment.line_position().is_end_of_line() {
// Is the operator on its own line.
if locator.contains_line_break(TextRange::new(
if source.contains_line_break(TextRange::new(
binary_expression.left.end(),
operator_offset,
)) && locator.contains_line_break(TextRange::new(
)) && source.contains_line_break(TextRange::new(
operator_offset,
binary_expression.right.start(),
)) {
@ -893,7 +886,7 @@ fn handle_trailing_binary_expression_left_or_operator_comment<'a>(
/// ```
fn handle_trailing_binary_like_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(
comment.enclosing_node().is_expr_bool_op() || comment.enclosing_node().is_expr_compare()
@ -908,7 +901,7 @@ fn handle_trailing_binary_like_comment<'a>(
let between_operands_range = TextRange::new(left_operand.end(), right_operand.start());
let mut tokens = SimpleTokenizer::new(locator.contents(), between_operands_range)
let mut tokens = SimpleTokenizer::new(source, between_operands_range)
.skip_trivia()
.skip_while(|token| token.kind == SimpleTokenKind::RParen);
let operator_offset = tokens
@ -990,7 +983,7 @@ fn handle_trailing_module_comment<'a>(
/// a trailing comment of the previous statement.
fn handle_module_level_own_line_comment_before_class_or_function_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(comment.enclosing_node().is_module());
// Only applies for own line comments on the module level...
@ -1013,7 +1006,7 @@ fn handle_module_level_own_line_comment_before_class_or_function_comment<'a>(
}
// Make the comment a leading comment if there's no empty line between the comment and the function / class header
if max_empty_lines(locator.slice(TextRange::new(comment.end(), following.start()))) == 0 {
if max_empty_lines(&source[TextRange::new(comment.end(), following.start())]) == 0 {
CommentPlacement::leading(following, comment)
} else {
// Otherwise attach the comment as trailing comment to the previous statement
@ -1034,7 +1027,7 @@ fn handle_slice_comments<'a>(
comment: DecoratedComment<'a>,
expr_slice: &'a ast::ExprSlice,
comment_ranges: &CommentRanges,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let ast::ExprSlice {
range: _,
@ -1045,7 +1038,7 @@ fn handle_slice_comments<'a>(
// Check for `foo[ # comment`, but only if they are on the same line
let after_lbracket = matches!(
BackwardsTokenizer::up_to(comment.start(), locator.contents(), comment_ranges)
BackwardsTokenizer::up_to(comment.start(), source, comment_ranges)
.skip_trivia()
.next(),
Some(SimpleToken {
@ -1069,7 +1062,7 @@ fn handle_slice_comments<'a>(
return CommentPlacement::dangling(comment.enclosing_node(), comment);
}
let assignment = assign_comment_in_slice(comment.range(), locator.contents(), expr_slice);
let assignment = assign_comment_in_slice(comment.range(), source, expr_slice);
let node = match assignment {
ExprSliceCommentSection::Lower => lower,
ExprSliceCommentSection::Upper => upper,
@ -1155,7 +1148,7 @@ fn handle_leading_class_with_decorators_comment<'a>(
fn handle_keyword_comment<'a>(
comment: DecoratedComment<'a>,
keyword: &'a ast::Keyword,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let start = keyword.arg.as_ref().map_or(keyword.start(), Ranged::end);
@ -1167,8 +1160,7 @@ fn handle_keyword_comment<'a>(
// )
// )
// ```
let mut tokenizer =
SimpleTokenizer::new(locator.contents(), TextRange::new(start, comment.start()));
let mut tokenizer = SimpleTokenizer::new(source, TextRange::new(start, comment.start()));
if tokenizer.any(|token| token.kind == SimpleTokenKind::LParen) {
return CommentPlacement::Default(comment);
}
@ -1188,7 +1180,7 @@ fn handle_keyword_comment<'a>(
fn handle_pattern_keyword_comment<'a>(
comment: DecoratedComment<'a>,
pattern_keyword: &'a ast::PatternKeyword,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
// If the comment is parenthesized, it should be attached to the value:
// ```python
@ -1199,7 +1191,7 @@ fn handle_pattern_keyword_comment<'a>(
// )
// ```
let mut tokenizer = SimpleTokenizer::new(
locator.contents(),
source,
TextRange::new(pattern_keyword.attr.end(), comment.start()),
);
if tokenizer.any(|token| token.kind == SimpleTokenKind::LParen) {
@ -1221,7 +1213,7 @@ fn handle_pattern_keyword_comment<'a>(
/// ```
fn handle_dict_unpacking_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(matches!(comment.enclosing_node(), AnyNodeRef::ExprDict(_)));
@ -1236,12 +1228,9 @@ fn handle_dict_unpacking_comment<'a>(
Some(preceding) => preceding.end(),
None => comment.enclosing_node().start(),
};
let mut tokens = SimpleTokenizer::new(
locator.contents(),
TextRange::new(preceding_end, comment.start()),
)
.skip_trivia()
.skip_while(|token| token.kind == SimpleTokenKind::RParen);
let mut tokens = SimpleTokenizer::new(source, TextRange::new(preceding_end, comment.start()))
.skip_trivia()
.skip_while(|token| token.kind == SimpleTokenKind::RParen);
// if the remaining tokens from the previous node are exactly `**`,
// re-assign the comment to the one that follows the stars.
@ -1264,7 +1253,7 @@ fn handle_dict_unpacking_comment<'a>(
/// ```
fn handle_key_value_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(matches!(
comment.enclosing_node(),
@ -1284,10 +1273,7 @@ fn handle_key_value_comment<'a>(
// }
// ```
// This prevents against detecting comments on starred expressions as key-value comments.
let tokens = SimpleTokenizer::new(
locator.contents(),
TextRange::new(preceding.end(), following.start()),
);
let tokens = SimpleTokenizer::new(source, TextRange::new(preceding.end(), following.start()));
if tokens
.skip_trivia()
.any(|token| token.kind == SimpleTokenKind::Colon)
@ -1334,7 +1320,7 @@ fn handle_call_comment(comment: DecoratedComment) -> CommentPlacement {
fn handle_attribute_comment<'a>(
comment: DecoratedComment<'a>,
attribute: &'a ast::ExprAttribute,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if comment.preceding_node().is_none() {
// ```text
@ -1365,7 +1351,7 @@ fn handle_attribute_comment<'a>(
// .attribute
// )
// ```
if let Some(right_paren) = SimpleTokenizer::starts_at(attribute.value.end(), locator.contents())
if let Some(right_paren) = SimpleTokenizer::starts_at(attribute.value.end(), source)
.skip_trivia()
.take_while(|token| token.kind == SimpleTokenKind::RParen)
.last()
@ -1398,7 +1384,7 @@ fn handle_attribute_comment<'a>(
let dot_token = find_only_token_in_range(
TextRange::new(attribute.value.end(), attribute.attr.start()),
SimpleTokenKind::Dot,
locator.contents(),
source,
);
if comment.end() < dot_token.start() {
return CommentPlacement::trailing(attribute.value.as_ref(), comment);
@ -1426,7 +1412,7 @@ fn handle_attribute_comment<'a>(
fn handle_expr_if_comment<'a>(
comment: DecoratedComment<'a>,
expr_if: &'a ast::ExprIf,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let ast::ExprIf {
range: _,
@ -1442,7 +1428,7 @@ fn handle_expr_if_comment<'a>(
let if_token = find_only_token_in_range(
TextRange::new(body.end(), test.start()),
SimpleTokenKind::If,
locator.contents(),
source,
);
// Between `if` and `test`
if if_token.start() < comment.start() && comment.start() < test.start() {
@ -1452,7 +1438,7 @@ fn handle_expr_if_comment<'a>(
let else_token = find_only_token_in_range(
TextRange::new(test.end(), orelse.start()),
SimpleTokenKind::Else,
locator.contents(),
source,
);
// Between `else` and `orelse`
if else_token.start() < comment.start() && comment.start() < orelse.start() {
@ -1477,13 +1463,11 @@ fn handle_expr_if_comment<'a>(
fn handle_trailing_expression_starred_star_end_of_line_comment<'a>(
comment: DecoratedComment<'a>,
starred: &'a ast::ExprStarred,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if comment.following_node().is_some() {
let tokenizer = SimpleTokenizer::new(
locator.contents(),
TextRange::new(starred.start(), comment.start()),
);
let tokenizer =
SimpleTokenizer::new(source, TextRange::new(starred.start(), comment.start()));
if !tokenizer
.skip_trivia()
.any(|token| token.kind() == SimpleTokenKind::LParen)
@ -1508,7 +1492,7 @@ fn handle_trailing_expression_starred_star_end_of_line_comment<'a>(
/// ```
fn handle_with_item_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(comment.enclosing_node().is_with_item());
@ -1522,7 +1506,7 @@ fn handle_with_item_comment<'a>(
let as_token = find_only_token_in_range(
TextRange::new(context_expr.end(), optional_vars.start()),
SimpleTokenKind::As,
locator.contents(),
source,
);
if comment.end() < as_token.start() {
@ -1567,7 +1551,7 @@ fn handle_pattern_match_class_comment<'a>(
/// ```
fn handle_pattern_match_as_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(comment.enclosing_node().is_pattern_match_as());
@ -1575,7 +1559,7 @@ fn handle_pattern_match_as_comment<'a>(
return CommentPlacement::Default(comment);
};
let mut tokens = SimpleTokenizer::starts_at(pattern.end(), locator.contents())
let mut tokens = SimpleTokenizer::starts_at(pattern.end(), source)
.skip_trivia()
.skip_while(|token| token.kind == SimpleTokenKind::RParen);
@ -1625,7 +1609,7 @@ fn handle_pattern_match_star_comment(comment: DecoratedComment) -> CommentPlacem
fn handle_pattern_match_mapping_comment<'a>(
comment: DecoratedComment<'a>,
pattern: &'a ast::PatternMatchMapping,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
// The `**` has to come at the end, so there can't be another node after it. (The identifier,
// like `rest` above, isn't a node.)
@ -1649,11 +1633,8 @@ fn handle_pattern_match_mapping_comment<'a>(
Some(preceding) => preceding.end(),
None => comment.enclosing_node().start(),
};
let mut tokens = SimpleTokenizer::new(
locator.contents(),
TextRange::new(preceding_end, comment.start()),
)
.skip_trivia();
let mut tokens =
SimpleTokenizer::new(source, TextRange::new(preceding_end, comment.start())).skip_trivia();
// If the remaining tokens from the previous node include `**`, mark as a dangling comment.
if tokens.any(|token| token.kind == SimpleTokenKind::DoubleStar) {
@ -1682,7 +1663,7 @@ fn handle_pattern_match_mapping_comment<'a>(
/// ```
fn handle_named_expr_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
debug_assert!(comment.enclosing_node().is_expr_named());
@ -1693,7 +1674,7 @@ fn handle_named_expr_comment<'a>(
let colon_equal = find_only_token_in_range(
TextRange::new(target.end(), value.start()),
SimpleTokenKind::ColonEqual,
locator.contents(),
source,
);
if comment.end() < colon_equal.start() {
@ -1738,7 +1719,7 @@ fn handle_named_expr_comment<'a>(
fn handle_lambda_comment<'a>(
comment: DecoratedComment<'a>,
lambda: &'a ast::ExprLambda,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if let Some(parameters) = lambda.parameters.as_deref() {
// Comments between the `lambda` and the parameters are dangling on the lambda:
@ -1770,10 +1751,8 @@ fn handle_lambda_comment<'a>(
// )
// )
// ```
let tokenizer = SimpleTokenizer::new(
locator.contents(),
TextRange::new(parameters.end(), comment.start()),
);
let tokenizer =
SimpleTokenizer::new(source, TextRange::new(parameters.end(), comment.start()));
if tokenizer
.skip_trivia()
.any(|token| token.kind == SimpleTokenKind::LParen)
@ -1801,10 +1780,8 @@ fn handle_lambda_comment<'a>(
// )
// )
// ```
let tokenizer = SimpleTokenizer::new(
locator.contents(),
TextRange::new(lambda.start(), comment.start()),
);
let tokenizer =
SimpleTokenizer::new(source, TextRange::new(lambda.start(), comment.start()));
if tokenizer
.skip_trivia()
.any(|token| token.kind == SimpleTokenKind::LParen)
@ -1834,10 +1811,10 @@ fn handle_lambda_comment<'a>(
fn handle_unary_op_comment<'a>(
comment: DecoratedComment<'a>,
unary_op: &'a ast::ExprUnaryOp,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let mut tokenizer = SimpleTokenizer::new(
locator.contents(),
source,
TextRange::new(unary_op.start(), unary_op.operand.start()),
)
.skip_trivia();
@ -1883,12 +1860,12 @@ fn handle_unary_op_comment<'a>(
/// that it remains on the same line as open bracket.
fn handle_bracketed_end_of_line_comment<'a>(
comment: DecoratedComment<'a>,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
if comment.line_position().is_end_of_line() {
// Ensure that there are no tokens between the open bracket and the comment.
let mut lexer = SimpleTokenizer::new(
locator.contents(),
source,
TextRange::new(comment.enclosing_node().start(), comment.start()),
)
.skip_trivia();
@ -2006,7 +1983,7 @@ fn handle_with_comment<'a>(
fn handle_comprehension_comment<'a>(
comment: DecoratedComment<'a>,
comprehension: &'a Comprehension,
locator: &Locator,
source: &str,
) -> CommentPlacement<'a> {
let is_own_line = comment.line_position().is_own_line();
@ -2031,7 +2008,7 @@ fn handle_comprehension_comment<'a>(
let in_token = find_only_token_in_range(
TextRange::new(comprehension.target.end(), comprehension.iter.start()),
SimpleTokenKind::In,
locator.contents(),
source,
);
// Comments between the target and the `in`
@ -2094,7 +2071,7 @@ fn handle_comprehension_comment<'a>(
let if_token = find_only_token_in_range(
TextRange::new(last_end, if_node.start()),
SimpleTokenKind::If,
locator.contents(),
source,
);
if is_own_line {
if last_end < comment.start() && comment.start() < if_token.start() {

9
crates/ruff_python_formatter/src/comments/visitor.rs
View file

@ -9,7 +9,6 @@ use ruff_python_ast::{Mod, Stmt};
#[allow(clippy::wildcard_imports)]
use ruff_python_ast::visitor::source_order::*;
use ruff_python_trivia::{CommentLinePosition, CommentRanges};
use ruff_source_file::Locator;
use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::comments::node_key::NodeRefEqualityKey;
@ -531,12 +530,12 @@ pub(super) struct CommentsMapBuilder<'a> {
comments: CommentsMap<'a>,
/// We need those for backwards lexing
comment_ranges: &'a CommentRanges,
locator: Locator<'a>,
source: &'a str,
}
impl<'a> PushComment<'a> for CommentsMapBuilder<'a> {
fn push_comment(&mut self, placement: DecoratedComment<'a>) {
let placement = place_comment(placement, self.comment_ranges, &self.locator);
let placement = place_comment(placement, self.comment_ranges, self.source);
match placement {
CommentPlacement::Leading { node, comment } => {
self.push_leading_comment(node, comment);
@ -598,11 +597,11 @@ impl<'a> PushComment<'a> for CommentsMapBuilder<'a> {
}
impl<'a> CommentsMapBuilder<'a> {
pub(crate) fn new(locator: Locator<'a>, comment_ranges: &'a CommentRanges) -> Self {
pub(crate) fn new(source: &'a str, comment_ranges: &'a CommentRanges) -> Self {
Self {
comments: CommentsMap::default(),
comment_ranges,
locator,
source,
}
}

17
crates/ruff_python_formatter/src/context.rs
View file

@ -1,12 +1,13 @@
use crate::comments::Comments;
use crate::other::f_string_element::FStringExpressionElementContext;
use crate::PyFormatOptions;
use std::fmt::{Debug, Formatter};
use std::ops::{Deref, DerefMut};
use ruff_formatter::{Buffer, FormatContext, GroupId, IndentWidth, SourceCode};
use ruff_python_ast::str::Quote;
use ruff_python_parser::Tokens;
use ruff_source_file::Locator;
use std::fmt::{Debug, Formatter};
use std::ops::{Deref, DerefMut};
use crate::comments::Comments;
use crate::other::f_string_element::FStringExpressionElementContext;
use crate::PyFormatOptions;
pub struct PyFormatContext<'a> {
options: PyFormatOptions,
@ -51,10 +52,6 @@ impl<'a> PyFormatContext<'a> {
self.contents
}
pub(crate) fn locator(&self) -> Locator<'a> {
Locator::new(self.contents)
}
pub(crate) fn set_node_level(&mut self, level: NodeLevel) {
self.node_level = level;
}

18
crates/ruff_python_formatter/src/expression/expr_f_string.rs
View file

@ -1,6 +1,5 @@
use ruff_python_ast::{AnyNodeRef, ExprFString, StringLike};
use ruff_source_file::Locator;
use ruff_text_size::Ranged;
use ruff_text_size::TextSlice;
use crate::expression::parentheses::{
in_parentheses_only_group, NeedsParentheses, OptionalParentheses,
@ -18,11 +17,8 @@ impl FormatNodeRule<ExprFString> for FormatExprFString {
let ExprFString { value, .. } = item;
if let [f_string_part] = value.as_slice() {
FormatFStringPart::new(
f_string_part,
f_string_quoting(item, &f.context().locator()),
)
.fmt(f)
FormatFStringPart::new(f_string_part, f_string_quoting(item, f.context().source()))
.fmt(f)
} else {
// Always join fstrings that aren't parenthesized and thus, are always on a single line.
if !f.context().node_level().is_parenthesized() {
@ -73,9 +69,9 @@ impl NeedsParentheses for ExprFString {
}
}
pub(crate) fn f_string_quoting(f_string: &ExprFString, locator: &Locator) -> Quoting {
let unprefixed = locator
.slice(f_string.range())
pub(crate) fn f_string_quoting(f_string: &ExprFString, source: &str) -> Quoting {
let unprefixed = source
.slice(f_string)
.trim_start_matches(|c| c != '"' && c != '\'');
let triple_quoted = unprefixed.starts_with(r#"""""#) || unprefixed.starts_with(r"'''");
@ -84,7 +80,7 @@ pub(crate) fn f_string_quoting(f_string: &ExprFString, locator: &Locator) -> Quo
.elements()
.filter_map(|element| element.as_expression())
.any(|expression| {
let string_content = locator.slice(expression.range());
let string_content = source.slice(expression);
if triple_quoted {
string_content.contains(r#"""""#) || string_content.contains("'''")
} else {

14
crates/ruff_python_formatter/src/expression/expr_number_literal.rs
View file

@ -2,7 +2,7 @@ use std::borrow::Cow;
use ruff_python_ast::AnyNodeRef;
use ruff_python_ast::{ExprNumberLiteral, Number};
use ruff_text_size::{Ranged, TextSize};
use ruff_text_size::{Ranged, TextSize, TextSlice};
use crate::expression::parentheses::{NeedsParentheses, OptionalParentheses};
use crate::prelude::*;
@ -14,28 +14,26 @@ impl FormatNodeRule<ExprNumberLiteral> for FormatExprNumberLiteral {
fn fmt_fields(&self, item: &ExprNumberLiteral, f: &mut PyFormatter) -> FormatResult<()> {
match item.value {
Number::Int(_) => {
let range = item.range();
let content = f.context().locator().slice(range);
let content = f.context().source().slice(item);
let normalized = normalize_integer(content);
match normalized {
Cow::Borrowed(_) => source_text_slice(range).fmt(f),
Cow::Borrowed(_) => source_text_slice(item.range()).fmt(f),
Cow::Owned(normalized) => text(&normalized).fmt(f),
}
}
Number::Float(_) => {
let range = item.range();
let content = f.context().locator().slice(range);
let content = f.context().source().slice(item);
let normalized = normalize_floating_number(content);
match normalized {
Cow::Borrowed(_) => source_text_slice(range).fmt(f),
Cow::Borrowed(_) => source_text_slice(item.range()).fmt(f),
Cow::Owned(normalized) => text(&normalized).fmt(f),
}
}
Number::Complex { .. } => {
let range = item.range();
let content = f.context().locator().slice(range);
let content = f.context().source().slice(item);
let normalized = normalize_floating_number(content.trim_end_matches(['j', 'J']));
match normalized {

4
crates/ruff_python_formatter/src/lib.rs
View file

@ -8,7 +8,6 @@ use ruff_python_ast::AstNode;
use ruff_python_ast::Mod;
use ruff_python_parser::{parse, AsMode, ParseError, Parsed};
use ruff_python_trivia::CommentRanges;
use ruff_source_file::Locator;
use crate::comments::{
has_skip_comment, leading_comments, trailing_comments, Comments, SourceComment,
@ -127,10 +126,9 @@ pub fn format_module_ast<'a>(
) -> FormatResult<Formatted<PyFormatContext<'a>>> {
let source_code = SourceCode::new(source);
let comments = Comments::from_ast(parsed.syntax(), source_code, comment_ranges);
let locator = Locator::new(source);
let formatted = format!(
PyFormatContext::new(options, locator.contents(), comments, parsed.tokens()),
PyFormatContext::new(options, source, comments, parsed.tokens()),
[parsed.syntax().format()]
)?;
formatted

11
crates/ruff_python_formatter/src/other/f_string.rs
View file

@ -1,6 +1,7 @@
use ruff_formatter::write;
use ruff_python_ast::{AnyStringFlags, FString, StringFlags};
use ruff_source_file::Locator;
use ruff_source_file::LineRanges;
use ruff_text_size::Ranged;
use crate::prelude::*;
use crate::preview::is_f_string_formatting_enabled;
@ -27,8 +28,6 @@ impl<'a> FormatFString<'a> {
impl Format<PyFormatContext<'_>> for FormatFString<'_> {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let locator = f.context().locator();
// If the preview style is enabled, make the decision on what quotes to use locally for each
// f-string instead of globally for the entire f-string expression.
let quoting = if is_f_string_formatting_enabled(f.context()) {
@ -66,7 +65,7 @@ impl Format<PyFormatContext<'_>> for FormatFString<'_> {
let context = FStringContext::new(
string_kind,
FStringLayout::from_f_string(self.value, &locator),
FStringLayout::from_f_string(self.value, f.context().source()),
);
// Starting prefix and quote
@ -117,7 +116,7 @@ pub(crate) enum FStringLayout {
}
impl FStringLayout {
pub(crate) fn from_f_string(f_string: &FString, locator: &Locator) -> Self {
pub(crate) fn from_f_string(f_string: &FString, source: &str) -> Self {
// Heuristic: Allow breaking the f-string expressions across multiple lines
// only if there already is at least one multiline expression. This puts the
// control in the hands of the user to decide if they want to break the
@ -133,7 +132,7 @@ impl FStringLayout {
if f_string
.elements
.expressions()
.any(|expr| memchr::memchr2(b'\n', b'\r', locator.slice(expr).as_bytes()).is_some())
.any(|expr| source.contains_line_break(expr.range()))
{
Self::Multiline
} else {

4
crates/ruff_python_formatter/src/other/f_string_element.rs
View file

@ -5,7 +5,7 @@ use ruff_python_ast::{
AnyStringFlags, ConversionFlag, Expr, FStringElement, FStringExpressionElement,
FStringLiteralElement, StringFlags,
};
use ruff_text_size::Ranged;
use ruff_text_size::{Ranged, TextSlice};
use crate::comments::{dangling_open_parenthesis_comments, trailing_comments};
use crate::context::{FStringState, NodeLevel, WithFStringState, WithNodeLevel};
@ -60,7 +60,7 @@ impl<'a> FormatFStringLiteralElement<'a> {
impl Format<PyFormatContext<'_>> for FormatFStringLiteralElement<'_> {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let literal_content = f.context().locator().slice(self.element.range());
let literal_content = f.context().source().slice(self.element);
let normalized =
normalize_string(literal_content, 0, self.fstring_flags, false, false, true);
match &normalized {

9
crates/ruff_python_formatter/src/range.rs
View file

@ -10,7 +10,6 @@ use ruff_python_parser::{parse, AsMode};
use ruff_python_trivia::{
indentation_at_offset, BackwardsTokenizer, CommentRanges, SimpleToken, SimpleTokenKind,
};
use ruff_source_file::Locator;
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use crate::comments::Comments;
@ -300,8 +299,7 @@ fn narrow_range(
enclosing_node: AnyNodeRef,
context: &PyFormatContext,
) -> TextRange {
let locator = context.locator();
let enclosing_indent = indentation_at_offset(enclosing_node.start(), &locator)
let enclosing_indent = indentation_at_offset(enclosing_node.start(), context.source())
.expect("Expected enclosing to never be a same line body statement.");
let mut visitor = NarrowRange {
@ -513,7 +511,7 @@ impl NarrowRange<'_> {
// dedent the second line to 0 spaces and the `indent` then adds a 2 space indentation to match the indentation in the source.
// This is incorrect because the leading whitespace is the content of the string and not indentation, resulting in changed string content.
if let Some(indentation) =
indentation_at_offset(first_child.start(), &self.context.locator())
indentation_at_offset(first_child.start(), self.context.source())
{
let relative_indent = indentation.strip_prefix(self.enclosing_indent).unwrap();
let expected_indents = self.level;
@ -718,8 +716,7 @@ impl Format<PyFormatContext<'_>> for FormatEnclosingNode<'_> {
/// # Panics
/// If `offset` is outside of `source`.
fn indent_level(offset: TextSize, source: &str, options: &PyFormatOptions) -> Option<u16> {
let locator = Locator::new(source);
let indentation = indentation_at_offset(offset, &locator)?;
let indentation = indentation_at_offset(offset, source)?;
let level = match options.indent_style() {
IndentStyle::Tab => {

10
crates/ruff_python_formatter/src/string/docstring.rs
View file

@ -2,23 +2,22 @@
// "reStructuredText."
#![allow(clippy::doc_markdown)]
use itertools::Itertools;
use std::cmp::Ordering;
use std::sync::LazyLock;
use std::{borrow::Cow, collections::VecDeque};
use itertools::Itertools;
use regex::Regex;
use ruff_formatter::printer::SourceMapGeneration;
use ruff_python_ast::{str::Quote, AnyStringFlags, StringFlags};
use ruff_python_trivia::CommentRanges;
use {
ruff_formatter::{write, FormatOptions, IndentStyle, LineWidth, Printed},
ruff_python_trivia::{is_python_whitespace, PythonWhitespace},
ruff_source_file::Locator,
ruff_text_size::{Ranged, TextLen, TextRange, TextSize},
};
use super::NormalizedString;
use crate::preview::{
is_docstring_code_block_in_docstring_indent_enabled,
is_join_implicit_concatenated_string_enabled,
@ -26,6 +25,8 @@ use crate::preview::{
use crate::string::StringQuotes;
use crate::{prelude::*, DocstringCodeLineWidth, FormatModuleError};
use super::NormalizedString;
/// Format a docstring by trimming whitespace and adjusting the indentation.
///
/// Summary of changes we make:
@ -1592,9 +1593,8 @@ fn docstring_format_source(
let comment_ranges = CommentRanges::from(parsed.tokens());
let source_code = ruff_formatter::SourceCode::new(source);
let comments = crate::Comments::from_ast(parsed.syntax(), source_code, &comment_ranges);
let locator = Locator::new(source);
let ctx = PyFormatContext::new(options, locator.contents(), comments, parsed.tokens())
let ctx = PyFormatContext::new(options, source, comments, parsed.tokens())
.in_docstring(docstring_quote_style);
let formatted = crate::format!(ctx, [parsed.syntax().format()])?;
formatted

28
crates/ruff_python_formatter/src/string/implicit.rs
View file

@ -6,6 +6,7 @@ use ruff_python_ast::str_prefix::{
AnyStringPrefix, ByteStringPrefix, FStringPrefix, StringLiteralPrefix,
};
use ruff_python_ast::{AnyStringFlags, FStringElement, StringFlags, StringLike, StringLikePart};
use ruff_source_file::LineRanges;
use ruff_text_size::{Ranged, TextRange};
use crate::comments::{leading_comments, trailing_comments};
@ -72,7 +73,7 @@ impl<'a> FormatImplicitConcatenatedStringExpanded<'a> {
impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedStringExpanded<'_> {
fn fmt(&self, f: &mut Formatter<PyFormatContext<'_>>) -> FormatResult<()> {
let comments = f.context().comments().clone();
let quoting = self.string.quoting(&f.context().locator());
let quoting = self.string.quoting(f.context().source());
let join_implicit_concatenated_string_enabled =
is_join_implicit_concatenated_string_enabled(f.context());
@ -158,10 +159,9 @@ impl<'a> FormatImplicitConcatenatedStringFlat<'a> {
if let StringLikePart::FString(fstring) = part {
if fstring.elements.iter().any(|element| match element {
// Same as for other literals. Multiline literals can't fit on a single line.
FStringElement::Literal(literal) => context
.locator()
.slice(literal.range())
.contains(['\n', '\r']),
FStringElement::Literal(literal) => {
context.source().contains_line_break(literal.range())
}
FStringElement::Expression(expression) => {
if is_f_string_formatting_enabled(context) {
// Expressions containing comments can't be joined.
@ -169,7 +169,7 @@ impl<'a> FormatImplicitConcatenatedStringFlat<'a> {
} else {
// Multiline f-string expressions can't be joined if the f-string formatting is disabled because
// the string gets inserted in verbatim preserving the newlines.
context.locator().slice(expression).contains(['\n', '\r'])
context.source().contains_line_break(expression.range())
}
}
}) {
@ -269,12 +269,7 @@ impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedStringFlat<'_> {
for part in self.string.parts().rev() {
assert!(part.is_string_literal());
if f.context()
.locator()
.slice(part.content_range())
.trim()
.is_empty()
{
if f.context().source()[part.content_range()].trim().is_empty() {
// Don't format the part.
parts.next_back();
} else {
@ -298,10 +293,7 @@ impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedStringFlat<'_> {
.fmt(f)?;
if first_non_empty {
first_non_empty = f
.context()
.locator()
.slice(part.content_range())
first_non_empty = f.context().source()[part.content_range()]
.trim_start()
.is_empty();
}
@ -328,7 +320,7 @@ impl Format<PyFormatContext<'_>> for FormatImplicitConcatenatedStringFlat<'_> {
self.flags,
FStringLayout::from_f_string(
f_string,
&f.context().locator(),
f.context().source(),
),
);
@ -365,7 +357,7 @@ struct FormatLiteralContent {
impl Format<PyFormatContext<'_>> for FormatLiteralContent {
fn fmt(&self, f: &mut PyFormatter) -> FormatResult<()> {
let content = f.context().locator().slice(self.range);
let content = &f.context().source()[self.range];
let mut normalized = normalize_string(
content,
0,

7
crates/ruff_python_formatter/src/string/mod.rs
View file

@ -7,7 +7,6 @@ use ruff_python_ast::{
str_prefix::{AnyStringPrefix, StringLiteralPrefix},
AnyStringFlags, StringFlags,
};
use ruff_source_file::Locator;
use ruff_text_size::Ranged;
use crate::expression::expr_f_string::f_string_quoting;
@ -89,16 +88,16 @@ impl From<Quote> for QuoteStyle {
// Extension trait that adds formatter specific helper methods to `StringLike`.
pub(crate) trait StringLikeExtensions {
fn quoting(&self, locator: &Locator<'_>) -> Quoting;
fn quoting(&self, source: &str) -> Quoting;
fn is_multiline(&self, source: &str) -> bool;
}
impl StringLikeExtensions for ast::StringLike<'_> {
fn quoting(&self, locator: &Locator<'_>) -> Quoting {
fn quoting(&self, source: &str) -> Quoting {
match self {
Self::String(_) | Self::Bytes(_) => Quoting::CanChange,
Self::FString(f_string) => f_string_quoting(f_string, locator),
Self::FString(f_string) => f_string_quoting(f_string, source),
}
}

21
crates/ruff_python_formatter/src/string/normalize.rs
View file

@ -7,7 +7,7 @@ use ruff_python_ast::visitor::source_order::SourceOrderVisitor;
use ruff_python_ast::{
str::Quote, AnyStringFlags, BytesLiteral, FString, StringFlags, StringLikePart, StringLiteral,
};
use ruff_text_size::{Ranged, TextRange};
use ruff_text_size::{Ranged, TextRange, TextSlice};
use crate::context::FStringState;
use crate::prelude::*;
@ -152,7 +152,7 @@ impl<'a, 'src> StringNormalizer<'a, 'src> {
/// Computes the strings preferred quotes.
pub(crate) fn choose_quotes(&self, string: StringLikePart) -> QuoteSelection {
let raw_content = self.context.locator().slice(string.content_range());
let raw_content = &self.context.source()[string.content_range()];
let first_quote_or_normalized_char_offset = raw_content
.bytes()
.position(|b| matches!(b, b'\\' | b'"' | b'\'' | b'\r' | b'{'));
@ -196,7 +196,7 @@ impl<'a, 'src> StringNormalizer<'a, 'src> {
/// Computes the strings preferred quotes and normalizes its content.
pub(crate) fn normalize(&self, string: StringLikePart) -> NormalizedString<'src> {
let raw_content = self.context.locator().slice(string.content_range());
let raw_content = &self.context.source()[string.content_range()];
let quote_selection = self.choose_quotes(string);
let normalized = if let Some(first_quote_or_escape_offset) =
@ -256,7 +256,7 @@ impl QuoteMetadata {
) -> Self {
match part {
StringLikePart::String(_) | StringLikePart::Bytes(_) => {
let text = context.locator().slice(part.content_range());
let text = &context.source()[part.content_range()];
Self::from_str(text, part.flags(), preferred_quote)
}
@ -277,7 +277,7 @@ impl QuoteMetadata {
};
let mut metadata = QuoteMetadata::from_str(
context.locator().slice(first.range()),
context.source().slice(first),
fstring.flags.into(),
preferred_quote,
);
@ -285,7 +285,7 @@ impl QuoteMetadata {
for literal in literals {
metadata = metadata
.merge(&QuoteMetadata::from_str(
context.locator().slice(literal.range()),
context.source().slice(literal),
fstring.flags.into(),
preferred_quote,
))
@ -294,7 +294,7 @@ impl QuoteMetadata {
metadata
} else {
let text = context.locator().slice(part.content_range());
let text = &context.source()[part.content_range()];
Self::from_str(text, part.flags(), preferred_quote)
}
@ -893,7 +893,7 @@ pub(super) fn is_fstring_with_quoted_debug_expression(
) -> bool {
if fstring.elements.expressions().any(|expression| {
if expression.debug_text.is_some() {
let content = context.locator().slice(expression.range());
let content = context.source().slice(expression);
match fstring.flags.quote_style() {
Quote::Single => {
if fstring.flags.is_triple_quoted() {
@ -969,10 +969,7 @@ pub(super) fn is_fstring_with_triple_quoted_literal_expression_containing_quotes
}
fn contains_quote(&self, range: TextRange, flags: AnyStringFlags) -> bool {
self.context
.locator()
.slice(range)
.contains(flags.quote_style().as_char())
self.context.source()[range].contains(flags.quote_style().as_char())
}
}

6
crates/ruff_python_formatter/src/verbatim.rs
View file

@ -7,7 +7,7 @@ use ruff_python_ast::AnyNodeRef;
use ruff_python_ast::Stmt;
use ruff_python_parser::{self as parser, TokenKind};
use ruff_python_trivia::lines_before;
use ruff_source_file::Locator;
use ruff_source_file::LineRanges;
use ruff_text_size::{Ranged, TextRange, TextSize};
use crate::comments::format::{empty_lines, format_comment};
@ -647,7 +647,7 @@ struct Indentation(u32);
impl Indentation {
fn from_stmt(stmt: &Stmt, source: &str) -> Indentation {
let line_start = Locator::new(source).line_start(stmt.start());
let line_start = source.line_start(stmt.start());
let mut indentation = 0u32;
for c in source[TextRange::new(line_start, stmt.start())].chars() {
@ -878,7 +878,7 @@ impl Format<PyFormatContext<'_>> for VerbatimText {
},
)));
match normalize_newlines(f.context().locator().slice(self.verbatim_range), ['\r']) {
match normalize_newlines(&f.context().source()[self.verbatim_range], ['\r']) {
Cow::Borrowed(_) => {
write!(f, [source_text_slice(self.verbatim_range)])?;
}