language-servers/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-09-26 11:59:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Use CommentRanges in backwards lexing (#7360)

Browse source 
## Summary

The tokenizer was split into a forward and a backwards tokenizer. The
backwards tokenizer uses the same names as the forwards ones (e.g.
`next_token`). The backwards tokenizer gets the comment ranges that we
already built to skip comments.

---------

Co-authored-by: Micha Reiser <micha@reiser.io>
This commit is contained in:
konsti 2023-09-16 05:21:45 +02:00 • committed by GitHub
parent 1f6e1485f9
commit 2cbe1733c8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
41 changed files with 744 additions and 628 deletions
Download patch file Download diff file Expand all files Collapse all files

57
crates/ruff_python_ast/tests/parenthesize.rs
View file

@ -1,5 +1,6 @@
use ruff_python_ast::parenthesize::parenthesized_range;
use ruff_python_parser::parse_expression;
use ruff_python_trivia::CommentRanges;
use ruff_text_size::TextRange;
#[test]
@ -10,7 +11,12 @@ fn test_parenthesized_name() {
let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref();
let parenthesized = parenthesized_range(name.into(), bin_op.into(), source_code);
let parenthesized = parenthesized_range(
name.into(),
bin_op.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, Some(TextRange::new(0.into(), 3.into())));
}
@ -22,7 +28,12 @@ fn test_non_parenthesized_name() {
let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref();
let parenthesized = parenthesized_range(name.into(), bin_op.into(), source_code);
let parenthesized = parenthesized_range(
name.into(),
bin_op.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, None);
}
@ -35,7 +46,12 @@ fn test_parenthesized_argument() {
let arguments = &call.arguments;
let argument = arguments.args.first().unwrap();
let parenthesized = parenthesized_range(argument.into(), arguments.into(), source_code);
let parenthesized = parenthesized_range(
argument.into(),
arguments.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, Some(TextRange::new(2.into(), 5.into())));
}
@ -48,7 +64,12 @@ fn test_non_parenthesized_argument() {
let arguments = &call.arguments;
let argument = arguments.args.first().unwrap();
let parenthesized = parenthesized_range(argument.into(), arguments.into(), source_code);
let parenthesized = parenthesized_range(
argument.into(),
arguments.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, None);
}
@ -60,7 +81,12 @@ fn test_parenthesized_tuple_member() {
let tuple = expr.as_tuple_expr().unwrap();
let member = tuple.elts.last().unwrap();
let parenthesized = parenthesized_range(member.into(), tuple.into(), source_code);
let parenthesized = parenthesized_range(
member.into(),
tuple.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, Some(TextRange::new(4.into(), 7.into())));
}
@ -72,7 +98,12 @@ fn test_non_parenthesized_tuple_member() {
let tuple = expr.as_tuple_expr().unwrap();
let member = tuple.elts.last().unwrap();
let parenthesized = parenthesized_range(member.into(), tuple.into(), source_code);
let parenthesized = parenthesized_range(
member.into(),
tuple.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, None);
}
@ -84,7 +115,12 @@ fn test_twice_parenthesized_name() {
let bin_op = expr.as_bin_op_expr().unwrap();
let name = bin_op.left.as_ref();
let parenthesized = parenthesized_range(name.into(), bin_op.into(), source_code);
let parenthesized = parenthesized_range(
name.into(),
bin_op.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, Some(TextRange::new(0.into(), 5.into())));
}
@ -97,6 +133,11 @@ fn test_twice_parenthesized_argument() {
let arguments = &call.arguments;
let argument = arguments.args.first().unwrap();
let parenthesized = parenthesized_range(argument.into(), arguments.into(), source_code);
let parenthesized = parenthesized_range(
argument.into(),
arguments.into(),
&CommentRanges::default(),
source_code,
);
assert_eq!(parenthesized, Some(TextRange::new(2.into(), 11.into())));
}