mirror of
https://github.com/astral-sh/ruff.git
synced 2025-09-26 20:09:22 +00:00
025768d303
## Summary Alternative to #11237 This PR adds a new `Tokens` struct which is a newtype wrapper around a vector of lexer output. This allows us to add a `kinds` method which returns an iterator over the corresponding `TokenKind`. This iterator is implemented as a separate `TokenKindIter` struct to allow using the type and provide additional methods like `peek` directly on the iterator. This exposes the linter to access the stream of `TokenKind` instead of `Tok`. Edit: I've made the necessary downstream changes and plan to merge the entire stack at once.
150 lines
3.7 KiB
Rust
150 lines
3.7 KiB
Rust
use ruff_python_index::Indexer;
|
|
use ruff_python_parser::{tokenize, Mode};
|
|
use ruff_source_file::Locator;
|
|
use ruff_text_size::TextSize;
|
|
|
|
#[test]
|
|
fn block_comments_two_line_block_at_start() {
|
|
// arrange
|
|
let source = "# line 1\n# line 2\n";
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(block_comments, vec![TextSize::new(0), TextSize::new(9)]);
|
|
}
|
|
|
|
#[test]
|
|
fn block_comments_indented_block() {
|
|
// arrange
|
|
let source = " # line 1\n # line 2\n";
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(block_comments, vec![TextSize::new(4), TextSize::new(17)]);
|
|
}
|
|
|
|
#[test]
|
|
fn block_comments_single_line_is_not_a_block() {
|
|
// arrange
|
|
let source = "\n";
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(block_comments, Vec::<TextSize>::new());
|
|
}
|
|
|
|
#[test]
|
|
fn block_comments_lines_with_code_not_a_block() {
|
|
// arrange
|
|
let source = "x = 1 # line 1\ny = 2 # line 2\n";
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(block_comments, Vec::<TextSize>::new());
|
|
}
|
|
|
|
#[test]
|
|
fn block_comments_sequential_lines_not_in_block() {
|
|
// arrange
|
|
let source = " # line 1\n # line 2\n";
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(block_comments, Vec::<TextSize>::new());
|
|
}
|
|
|
|
#[test]
|
|
fn block_comments_lines_in_triple_quotes_not_a_block() {
|
|
// arrange
|
|
let source = r#"
|
|
"""
|
|
# line 1
|
|
# line 2
|
|
"""
|
|
"#;
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(block_comments, Vec::<TextSize>::new());
|
|
}
|
|
|
|
#[test]
|
|
fn block_comments_stress_test() {
|
|
// arrange
|
|
let source = r#"
|
|
# block comment 1 line 1
|
|
# block comment 2 line 2
|
|
|
|
# these lines
|
|
# do not form
|
|
# a block comment
|
|
|
|
x = 1 # these lines also do not
|
|
y = 2 # do not form a block comment
|
|
|
|
# these lines do form a block comment
|
|
#
|
|
|
|
#
|
|
# and so do these
|
|
#
|
|
|
|
"""
|
|
# these lines are in triple quotes and
|
|
# therefore do not form a block comment
|
|
"""
|
|
"#;
|
|
let tokens = tokenize(source, Mode::Module);
|
|
let locator = Locator::new(source);
|
|
let indexer = Indexer::from_tokens(&tokens, &locator);
|
|
|
|
// act
|
|
let block_comments = indexer.comment_ranges().block_comments(&locator);
|
|
|
|
// assert
|
|
assert_eq!(
|
|
block_comments,
|
|
vec![
|
|
// Block #1
|
|
TextSize::new(1),
|
|
TextSize::new(26),
|
|
// Block #2
|
|
TextSize::new(174),
|
|
TextSize::new(212),
|
|
// Block #3
|
|
TextSize::new(219),
|
|
TextSize::new(225),
|
|
TextSize::new(247)
|
|
]
|
|
);
|
|
}
|