language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-16 01:25:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Use CommentRanges in backwards lexing (#7360)

Browse source 
## Summary

The tokenizer was split into a forward and a backwards tokenizer. The
backwards tokenizer uses the same names as the forwards ones (e.g.
`next_token`). The backwards tokenizer gets the comment ranges that we
already built to skip comments.

---------

Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
konsti 2023-09-16 05:21:45 +02:00 • committed by GitHub
parent 1f6e1485f9
commit 2cbe1733c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 744 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

24
crates/ruff_python_formatter/src/pattern/mod.rs
View file

@ -1,7 +1,10 @@
use ruff_formatter::{FormatOwnedWithRule, FormatRefWithRule, FormatRule, FormatRuleWithOptions};
use ruff_python_ast::node::AnyNodeRef;
use ruff_python_ast::Pattern;
use ruff_python_trivia::{first_non_trivia_token, SimpleToken, SimpleTokenKind, SimpleTokenizer};
use ruff_python_trivia::CommentRanges;
use ruff_python_trivia::{
first_non_trivia_token, BackwardsTokenizer, SimpleToken, SimpleTokenKind,
};
use ruff_text_size::Ranged;
use crate::expression::parentheses::{
@ -48,7 +51,11 @@ impl FormatRule<Pattern, PyFormatContext<'_>> for FormatPattern {
});
let parenthesize = match self.parentheses {
Parentheses::Preserve => is_pattern_parenthesized(pattern, f.context().source()),
Parentheses::Preserve => is_pattern_parenthesized(
pattern,
f.context().comments().ranges(),
f.context().source(),
),
Parentheses::Always => true,
Parentheses::Never => false,
};
@ -98,7 +105,11 @@ impl<'ast> IntoFormat<PyFormatContext<'ast>> for Pattern {
}
}
fn is_pattern_parenthesized(pattern: &Pattern, contents: &str) -> bool {
fn is_pattern_parenthesized(
pattern: &Pattern,
comment_ranges: &CommentRanges,
contents: &str,
) -> bool {
// First test if there's a closing parentheses because it tends to be cheaper.
if matches!(
first_non_trivia_token(pattern.end(), contents),
@ -107,11 +118,10 @@ fn is_pattern_parenthesized(pattern: &Pattern, contents: &str) -> bool {
..
})
) {
let mut tokenizer =
SimpleTokenizer::up_to_without_back_comment(pattern.start(), contents).skip_trivia();
matches!(
tokenizer.next_back(),
BackwardsTokenizer::up_to(pattern.start(), contents, comment_ranges)
.skip_trivia()
.next(),
Some(SimpleToken {
kind: SimpleTokenKind::LParen,
..