mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-30 07:07:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Use CommentRanges in backwards lexing (#7360)

Browse source 
## Summary

The tokenizer was split into a forward and a backwards tokenizer. The
backwards tokenizer uses the same names as the forwards ones (e.g.
`next_token`). The backwards tokenizer gets the comment ranges that we
already built to skip comments.

---------

Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
konsti 2023-09-16 05:21:45 +02:00 • committed by GitHub
parent 1f6e1485f9
commit 2cbe1733c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 744 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crates/ruff_python_ast/src/parenthesize.rs
View file

@ -1,5 +1,5 @@
use ruff_python_trivia::{SimpleTokenKind, SimpleTokenizer};
use ruff_text_size::{Ranged, TextRange, TextSize};
use ruff_python_trivia::{BackwardsTokenizer, CommentRanges, SimpleTokenKind, SimpleTokenizer};
use ruff_text_size::{Ranged, TextLen, TextRange};
use crate::node::AnyNodeRef;
use crate::ExpressionRef;
@ -9,6 +9,7 @@ use crate::ExpressionRef;
pub fn parenthesized_range(
expr: ExpressionRef,
parent: AnyNodeRef,
comment_ranges: &CommentRanges,
source: &str,
) -> Option<TextRange> {
// If the parent is a node that brings its own parentheses, exclude the closing parenthesis
@ -23,7 +24,7 @@ pub fn parenthesized_range(
// - `Tuple`: The elements of a tuple. The only risk is a single-element tuple (e.g., `(x,)`),
// which must have a trailing comma anyway.
let exclusive_parent_end = if parent.is_arguments() {
parent.end() - TextSize::new(1)
parent.end() - ")".text_len()
} else {
parent.end()
};
@ -33,9 +34,8 @@ pub fn parenthesized_range(
.skip_trivia()
.take_while(|token| token.kind == SimpleTokenKind::RParen);
let left_tokenizer = SimpleTokenizer::up_to_without_back_comment(expr.start(), source)
let left_tokenizer = BackwardsTokenizer::up_to(expr.start(), source, comment_ranges)
.skip_trivia()
.rev()
.take_while(|token| token.kind == SimpleTokenKind::LParen);
// Zip closing parenthesis with opening parenthesis. The order is intentional, as testing for