Fix SimpleTokenizer's backward lexing of # (#5878)

This commit is contained in:
Micha Reiser 2023-07-20 11:54:18 +02:00 committed by GitHub
parent 8c5f8a8aef
commit 76e9ce6dc0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 1863 additions and 52 deletions

View file

@ -8,8 +8,7 @@ use ruff_python_ast::node::{AnyNodeRef, AstNode};
use ruff_python_ast::source_code::Locator;
use ruff_python_ast::whitespace;
use ruff_python_trivia::{
first_non_trivia_token_rev, PythonWhitespace, SimpleToken, SimpleTokenKind, SimpleTokenizer,
UniversalNewlines,
PythonWhitespace, SimpleToken, SimpleTokenKind, SimpleTokenizer, UniversalNewlines,
};
use crate::comments::visitor::{CommentPlacement, DecoratedComment};
@ -1059,7 +1058,9 @@ fn handle_slice_comments<'a>(
// Check for `foo[ # comment`, but only if they are on the same line
let after_lbracket = matches!(
first_non_trivia_token_rev(comment.slice().start(), locator.contents()),
SimpleTokenizer::up_to_without_back_comment(comment.slice().start(), locator.contents())
.skip_trivia()
.next_back(),
Some(SimpleToken {
kind: SimpleTokenKind::LBracket,
..