Fix Indexer fails to identify continuation preceded by newline #10351 (#10354)

## Summary

Fixes #10351

It seems the bug was caused by this section of code

b669306c87/crates/ruff_python_index/src/indexer.rs (L55-L58)

It's true that newline tokens cannot be immediately followed by line
continuations, but only outside parentheses. e.g. the exception
```
(
    1
    \
    + 2)
```

But why was this check put there in the first place? Is it guarding
against something else?



## Test Plan

New test was added to indexer
This commit is contained in:
Auguste Lalande 2024-03-12 00:35:41 -04:00 committed by GitHub
parent b669306c87
commit dacec7377c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -37,7 +37,6 @@ impl Indexer {
let mut continuation_lines = Vec::new();
// Token, end
let mut prev_end = TextSize::default();
let mut prev_token: Option<&Tok> = None;
let mut line_start = TextSize::default();
for (tok, range) in tokens.iter().flatten() {
@ -51,11 +50,7 @@ impl Indexer {
if text == "\r" && trivia.as_bytes().get(index + 1) == Some(&b'\n') {
continue;
}
// Newlines after a newline never form a continuation.
if !matches!(prev_token, Some(Tok::Newline | Tok::NonLogicalNewline)) {
continuation_lines.push(line_start);
}
continuation_lines.push(line_start);
// SAFETY: Safe because of the len assertion at the top of the function.
#[allow(clippy::cast_possible_truncation)]
@ -80,7 +75,6 @@ impl Indexer {
_ => {}
}
prev_token = Some(tok);
prev_end = range.end();
}
@ -361,6 +355,33 @@ f'foo { 'str1' \
TextSize::new(63),
]
);
let contents = r"
x = (
1
\
\
\
\
+ 2)
"
.trim();
let lxr: Vec<LexResult> = lexer::lex(contents, Mode::Module).collect();
let indexer = Indexer::from_tokens(lxr.as_slice(), &Locator::new(contents));
assert_eq!(
indexer.continuation_line_starts(),
[
// row 3
TextSize::new(12),
// row 4
TextSize::new(18),
// row 5
TextSize::new(24),
// row 7
TextSize::new(31),
]
);
}
#[test]