mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-24 13:33:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Tokenizer: Emit only a single bogus token (#7425)

Browse source 
**Summary** Instead of emitting a bogus token per char, we now only emit
on single last bogus token. This leads to much more concise output.

**Test Plan** Updated fixtures
This commit is contained in:
konsti 2023-09-19 16:06:03 +02:00 • committed by GitHub
parent 97510c888b
commit 6dade5b9ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 19 additions and 1739 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/ruff_python_trivia/src/tokenizer.rs
View file

@ -504,10 +504,12 @@ impl<'a> SimpleTokenizer<'a> {
// Emit a single final bogus token
let token = SimpleToken {
kind: SimpleTokenKind::Bogus,
range: TextRange::at(self.offset, first.text_len()),
range: TextRange::new(self.offset, self.source.text_len()),
};
self.offset += first.text_len();
// Set the cursor to EOF
self.cursor = Cursor::new("");
self.offset = self.source.text_len();
return token;
}
@ -786,10 +788,12 @@ impl<'a> BackwardsTokenizer<'a> {
if self.bogus {
let token = SimpleToken {
kind: SimpleTokenKind::Bogus,
range: TextRange::at(self.back_offset - last.text_len(), last.text_len()),
range: TextRange::up_to(self.back_offset),
};
self.back_offset -= last.text_len();
// Set the cursor to EOF
self.cursor = Cursor::new("");
self.back_offset = TextSize::new(0);
return token;
}