Fix Indexer fails to identify continuation preceded by newline #10351 (#10354)

## Summary

Fixes #10351

It seems the bug was caused by this section of code

b669306c87/crates/ruff_python_index/src/indexer.rs (L55-L58)

It's true that newline tokens cannot be immediately followed by line
continuations, but only outside parentheses. e.g. the exception
```
(
    1
    \
    + 2)
```

But why was this check put there in the first place? Is it guarding
against something else?



## Test Plan

New test was added to indexer
This commit is contained in:
Auguste Lalande 2024-03-12 00:35:41 -04:00 committed by GitHub
parent b669306c87
commit dacec7377c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -37,7 +37,6 @@ impl Indexer {
let mut continuation_lines = Vec::new(); let mut continuation_lines = Vec::new();
// Token, end // Token, end
let mut prev_end = TextSize::default(); let mut prev_end = TextSize::default();
let mut prev_token: Option<&Tok> = None;
let mut line_start = TextSize::default(); let mut line_start = TextSize::default();
for (tok, range) in tokens.iter().flatten() { for (tok, range) in tokens.iter().flatten() {
@ -51,11 +50,7 @@ impl Indexer {
if text == "\r" && trivia.as_bytes().get(index + 1) == Some(&b'\n') { if text == "\r" && trivia.as_bytes().get(index + 1) == Some(&b'\n') {
continue; continue;
} }
continuation_lines.push(line_start);
// Newlines after a newline never form a continuation.
if !matches!(prev_token, Some(Tok::Newline | Tok::NonLogicalNewline)) {
continuation_lines.push(line_start);
}
// SAFETY: Safe because of the len assertion at the top of the function. // SAFETY: Safe because of the len assertion at the top of the function.
#[allow(clippy::cast_possible_truncation)] #[allow(clippy::cast_possible_truncation)]
@ -80,7 +75,6 @@ impl Indexer {
_ => {} _ => {}
} }
prev_token = Some(tok);
prev_end = range.end(); prev_end = range.end();
} }
@ -361,6 +355,33 @@ f'foo { 'str1' \
TextSize::new(63), TextSize::new(63),
] ]
); );
let contents = r"
x = (
1
\
\
\
\
+ 2)
"
.trim();
let lxr: Vec<LexResult> = lexer::lex(contents, Mode::Module).collect();
let indexer = Indexer::from_tokens(lxr.as_slice(), &Locator::new(contents));
assert_eq!(
indexer.continuation_line_starts(),
[
// row 3
TextSize::new(12),
// row 4
TextSize::new(18),
// row 5
TextSize::new(24),
// row 7
TextSize::new(31),
]
);
} }
#[test] #[test]