mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-02 18:02:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Tokenizer: Emit only a single bogus token (#7425)

Browse source 
**Summary** Instead of emitting a bogus token per char, we now only emit
on single last bogus token. This leads to much more concise output.

**Test Plan** Updated fixtures
This commit is contained in:
konsti 2023-09-19 16:06:03 +02:00 • committed by GitHub
parent 97510c888b
commit 6dade5b9ab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 19 additions and 1739 deletions
Download patch file Download diff file Expand all files Collapse all files

202
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap
View file

@ -13,206 +13,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 50..51,
},
SimpleToken {
kind: Bogus,
range: 49..50,
},
SimpleToken {
kind: Bogus,
range: 48..49,
},
SimpleToken {
kind: Bogus,
range: 47..48,
},
SimpleToken {
kind: Bogus,
range: 46..47,
},
SimpleToken {
kind: Bogus,
range: 45..46,
},
SimpleToken {
kind: Bogus,
range: 44..45,
},
SimpleToken {
kind: Bogus,
range: 43..44,
},
SimpleToken {
kind: Bogus,
range: 42..43,
},
SimpleToken {
kind: Bogus,
range: 41..42,
},
SimpleToken {
kind: Bogus,
range: 40..41,
},
SimpleToken {
kind: Bogus,
range: 39..40,
},
SimpleToken {
kind: Bogus,
range: 38..39,
},
SimpleToken {
kind: Bogus,
range: 37..38,
},
SimpleToken {
kind: Bogus,
range: 36..37,
},
SimpleToken {
kind: Bogus,
range: 35..36,
},
SimpleToken {
kind: Bogus,
range: 34..35,
},
SimpleToken {
kind: Bogus,
range: 33..34,
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 26..27,
},
SimpleToken {
kind: Bogus,
range: 25..26,
},
SimpleToken {
kind: Bogus,
range: 24..25,
},
SimpleToken {
kind: Bogus,
range: 23..24,
},
SimpleToken {
kind: Bogus,
range: 22..23,
},
SimpleToken {
kind: Bogus,
range: 21..22,
},
SimpleToken {
kind: Bogus,
range: 20..21,
},
SimpleToken {
kind: Bogus,
range: 19..20,
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..51,
},
]

58
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap
View file

@ -17,62 +17,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..15,
},
]

74
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap
View file

@ -17,78 +17,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..19,
},
]

6
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap
View file

@ -9,10 +9,6 @@ expression: test_case.tokens()
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 2..3,
range: 1..3,
},
]

302
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap
View file

@ -9,306 +9,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 75..76,
},
SimpleToken {
kind: Bogus,
range: 74..75,
},
SimpleToken {
kind: Bogus,
range: 73..74,
},
SimpleToken {
kind: Bogus,
range: 72..73,
},
SimpleToken {
kind: Bogus,
range: 71..72,
},
SimpleToken {
kind: Bogus,
range: 70..71,
},
SimpleToken {
kind: Bogus,
range: 69..70,
},
SimpleToken {
kind: Bogus,
range: 68..69,
},
SimpleToken {
kind: Bogus,
range: 67..68,
},
SimpleToken {
kind: Bogus,
range: 66..67,
},
SimpleToken {
kind: Bogus,
range: 65..66,
},
SimpleToken {
kind: Bogus,
range: 64..65,
},
SimpleToken {
kind: Bogus,
range: 63..64,
},
SimpleToken {
kind: Bogus,
range: 62..63,
},
SimpleToken {
kind: Bogus,
range: 61..62,
},
SimpleToken {
kind: Bogus,
range: 60..61,
},
SimpleToken {
kind: Bogus,
range: 59..60,
},
SimpleToken {
kind: Bogus,
range: 58..59,
},
SimpleToken {
kind: Bogus,
range: 57..58,
},
SimpleToken {
kind: Bogus,
range: 56..57,
},
SimpleToken {
kind: Bogus,
range: 55..56,
},
SimpleToken {
kind: Bogus,
range: 54..55,
},
SimpleToken {
kind: Bogus,
range: 53..54,
},
SimpleToken {
kind: Bogus,
range: 52..53,
},
SimpleToken {
kind: Bogus,
range: 51..52,
},
SimpleToken {
kind: Bogus,
range: 50..51,
},
SimpleToken {
kind: Bogus,
range: 49..50,
},
SimpleToken {
kind: Bogus,
range: 48..49,
},
SimpleToken {
kind: Bogus,
range: 47..48,
},
SimpleToken {
kind: Bogus,
range: 46..47,
},
SimpleToken {
kind: Bogus,
range: 45..46,
},
SimpleToken {
kind: Bogus,
range: 44..45,
},
SimpleToken {
kind: Bogus,
range: 43..44,
},
SimpleToken {
kind: Bogus,
range: 42..43,
},
SimpleToken {
kind: Bogus,
range: 41..42,
},
SimpleToken {
kind: Bogus,
range: 40..41,
},
SimpleToken {
kind: Bogus,
range: 39..40,
},
SimpleToken {
kind: Bogus,
range: 38..39,
},
SimpleToken {
kind: Bogus,
range: 37..38,
},
SimpleToken {
kind: Bogus,
range: 36..37,
},
SimpleToken {
kind: Bogus,
range: 35..36,
},
SimpleToken {
kind: Bogus,
range: 34..35,
},
SimpleToken {
kind: Bogus,
range: 33..34,
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 26..27,
},
SimpleToken {
kind: Bogus,
range: 25..26,
},
SimpleToken {
kind: Bogus,
range: 24..25,
},
SimpleToken {
kind: Bogus,
range: 23..24,
},
SimpleToken {
kind: Bogus,
range: 22..23,
},
SimpleToken {
kind: Bogus,
range: 21..22,
},
SimpleToken {
kind: Bogus,
range: 20..21,
},
SimpleToken {
kind: Bogus,
range: 19..20,
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..76,
},
]

310
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap
View file

@ -9,314 +9,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 77..78,
},
SimpleToken {
kind: Bogus,
range: 76..77,
},
SimpleToken {
kind: Bogus,
range: 75..76,
},
SimpleToken {
kind: Bogus,
range: 74..75,
},
SimpleToken {
kind: Bogus,
range: 73..74,
},
SimpleToken {
kind: Bogus,
range: 72..73,
},
SimpleToken {
kind: Bogus,
range: 71..72,
},
SimpleToken {
kind: Bogus,
range: 70..71,
},
SimpleToken {
kind: Bogus,
range: 69..70,
},
SimpleToken {
kind: Bogus,
range: 68..69,
},
SimpleToken {
kind: Bogus,
range: 67..68,
},
SimpleToken {
kind: Bogus,
range: 66..67,
},
SimpleToken {
kind: Bogus,
range: 65..66,
},
SimpleToken {
kind: Bogus,
range: 64..65,
},
SimpleToken {
kind: Bogus,
range: 63..64,
},
SimpleToken {
kind: Bogus,
range: 62..63,
},
SimpleToken {
kind: Bogus,
range: 61..62,
},
SimpleToken {
kind: Bogus,
range: 60..61,
},
SimpleToken {
kind: Bogus,
range: 59..60,
},
SimpleToken {
kind: Bogus,
range: 58..59,
},
SimpleToken {
kind: Bogus,
range: 57..58,
},
SimpleToken {
kind: Bogus,
range: 56..57,
},
SimpleToken {
kind: Bogus,
range: 55..56,
},
SimpleToken {
kind: Bogus,
range: 54..55,
},
SimpleToken {
kind: Bogus,
range: 53..54,
},
SimpleToken {
kind: Bogus,
range: 52..53,
},
SimpleToken {
kind: Bogus,
range: 51..52,
},
SimpleToken {
kind: Bogus,
range: 50..51,
},
SimpleToken {
kind: Bogus,
range: 49..50,
},
SimpleToken {
kind: Bogus,
range: 48..49,
},
SimpleToken {
kind: Bogus,
range: 47..48,
},
SimpleToken {
kind: Bogus,
range: 46..47,
},
SimpleToken {
kind: Bogus,
range: 45..46,
},
SimpleToken {
kind: Bogus,
range: 44..45,
},
SimpleToken {
kind: Bogus,
range: 43..44,
},
SimpleToken {
kind: Bogus,
range: 42..43,
},
SimpleToken {
kind: Bogus,
range: 41..42,
},
SimpleToken {
kind: Bogus,
range: 40..41,
},
SimpleToken {
kind: Bogus,
range: 39..40,
},
SimpleToken {
kind: Bogus,
range: 38..39,
},
SimpleToken {
kind: Bogus,
range: 37..38,
},
SimpleToken {
kind: Bogus,
range: 36..37,
},
SimpleToken {
kind: Bogus,
range: 35..36,
},
SimpleToken {
kind: Bogus,
range: 34..35,
},
SimpleToken {
kind: Bogus,
range: 33..34,
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 26..27,
},
SimpleToken {
kind: Bogus,
range: 25..26,
},
SimpleToken {
kind: Bogus,
range: 24..25,
},
SimpleToken {
kind: Bogus,
range: 23..24,
},
SimpleToken {
kind: Bogus,
range: 22..23,
},
SimpleToken {
kind: Bogus,
range: 21..22,
},
SimpleToken {
kind: Bogus,
range: 20..21,
},
SimpleToken {
kind: Bogus,
range: 19..20,
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..78,
},
]

202
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap
View file

@ -17,206 +17,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 50..51,
},
SimpleToken {
kind: Bogus,
range: 49..50,
},
SimpleToken {
kind: Bogus,
range: 48..49,
},
SimpleToken {
kind: Bogus,
range: 47..48,
},
SimpleToken {
kind: Bogus,
range: 46..47,
},
SimpleToken {
kind: Bogus,
range: 45..46,
},
SimpleToken {
kind: Bogus,
range: 44..45,
},
SimpleToken {
kind: Bogus,
range: 43..44,
},
SimpleToken {
kind: Bogus,
range: 42..43,
},
SimpleToken {
kind: Bogus,
range: 41..42,
},
SimpleToken {
kind: Bogus,
range: 40..41,
},
SimpleToken {
kind: Bogus,
range: 39..40,
},
SimpleToken {
kind: Bogus,
range: 38..39,
},
SimpleToken {
kind: Bogus,
range: 37..38,
},
SimpleToken {
kind: Bogus,
range: 36..37,
},
SimpleToken {
kind: Bogus,
range: 35..36,
},
SimpleToken {
kind: Bogus,
range: 34..35,
},
SimpleToken {
kind: Bogus,
range: 33..34,
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 26..27,
},
SimpleToken {
kind: Bogus,
range: 25..26,
},
SimpleToken {
kind: Bogus,
range: 24..25,
},
SimpleToken {
kind: Bogus,
range: 23..24,
},
SimpleToken {
kind: Bogus,
range: 22..23,
},
SimpleToken {
kind: Bogus,
range: 21..22,
},
SimpleToken {
kind: Bogus,
range: 20..21,
},
SimpleToken {
kind: Bogus,
range: 19..20,
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..51,
},
]

46
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap
View file

@ -17,50 +17,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..12,
},
]

130
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap
View file

@ -17,134 +17,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 26..27,
},
SimpleToken {
kind: Bogus,
range: 25..26,
},
SimpleToken {
kind: Bogus,
range: 24..25,
},
SimpleToken {
kind: Bogus,
range: 23..24,
},
SimpleToken {
kind: Bogus,
range: 22..23,
},
SimpleToken {
kind: Bogus,
range: 21..22,
},
SimpleToken {
kind: Bogus,
range: 20..21,
},
SimpleToken {
kind: Bogus,
range: 19..20,
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..33,
},
]

102
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap
View file

@ -21,106 +21,6 @@ expression: test_case.tokens()
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 33..34,
},
SimpleToken {
kind: Bogus,
range: 34..35,
},
SimpleToken {
kind: Bogus,
range: 35..36,
},
SimpleToken {
kind: Bogus,
range: 36..37,
},
SimpleToken {
kind: Bogus,
range: 37..38,
},
SimpleToken {
kind: Bogus,
range: 38..39,
},
SimpleToken {
kind: Bogus,
range: 39..40,
},
SimpleToken {
kind: Bogus,
range: 40..41,
},
SimpleToken {
kind: Bogus,
range: 41..42,
},
SimpleToken {
kind: Bogus,
range: 42..43,
},
SimpleToken {
kind: Bogus,
range: 43..44,
},
SimpleToken {
kind: Bogus,
range: 44..45,
},
SimpleToken {
kind: Bogus,
range: 45..46,
},
SimpleToken {
kind: Bogus,
range: 46..47,
},
SimpleToken {
kind: Bogus,
range: 47..48,
},
SimpleToken {
kind: Bogus,
range: 48..49,
},
SimpleToken {
kind: Bogus,
range: 49..50,
},
SimpleToken {
kind: Bogus,
range: 50..51,
},
SimpleToken {
kind: Bogus,
range: 51..52,
},
SimpleToken {
kind: Bogus,
range: 52..53,
range: 27..53,
},
]

314
crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap
View file

@ -9,318 +9,6 @@ expression: test_case.tokenize_reverse()
},
SimpleToken {
kind: Bogus,
range: 78..79,
},
SimpleToken {
kind: Bogus,
range: 77..78,
},
SimpleToken {
kind: Bogus,
range: 76..77,
},
SimpleToken {
kind: Bogus,
range: 75..76,
},
SimpleToken {
kind: Bogus,
range: 74..75,
},
SimpleToken {
kind: Bogus,
range: 73..74,
},
SimpleToken {
kind: Bogus,
range: 72..73,
},
SimpleToken {
kind: Bogus,
range: 71..72,
},
SimpleToken {
kind: Bogus,
range: 70..71,
},
SimpleToken {
kind: Bogus,
range: 69..70,
},
SimpleToken {
kind: Bogus,
range: 68..69,
},
SimpleToken {
kind: Bogus,
range: 67..68,
},
SimpleToken {
kind: Bogus,
range: 66..67,
},
SimpleToken {
kind: Bogus,
range: 65..66,
},
SimpleToken {
kind: Bogus,
range: 64..65,
},
SimpleToken {
kind: Bogus,
range: 63..64,
},
SimpleToken {
kind: Bogus,
range: 62..63,
},
SimpleToken {
kind: Bogus,
range: 61..62,
},
SimpleToken {
kind: Bogus,
range: 60..61,
},
SimpleToken {
kind: Bogus,
range: 59..60,
},
SimpleToken {
kind: Bogus,
range: 58..59,
},
SimpleToken {
kind: Bogus,
range: 57..58,
},
SimpleToken {
kind: Bogus,
range: 56..57,
},
SimpleToken {
kind: Bogus,
range: 55..56,
},
SimpleToken {
kind: Bogus,
range: 54..55,
},
SimpleToken {
kind: Bogus,
range: 53..54,
},
SimpleToken {
kind: Bogus,
range: 52..53,
},
SimpleToken {
kind: Bogus,
range: 51..52,
},
SimpleToken {
kind: Bogus,
range: 50..51,
},
SimpleToken {
kind: Bogus,
range: 49..50,
},
SimpleToken {
kind: Bogus,
range: 48..49,
},
SimpleToken {
kind: Bogus,
range: 47..48,
},
SimpleToken {
kind: Bogus,
range: 46..47,
},
SimpleToken {
kind: Bogus,
range: 45..46,
},
SimpleToken {
kind: Bogus,
range: 44..45,
},
SimpleToken {
kind: Bogus,
range: 43..44,
},
SimpleToken {
kind: Bogus,
range: 42..43,
},
SimpleToken {
kind: Bogus,
range: 41..42,
},
SimpleToken {
kind: Bogus,
range: 40..41,
},
SimpleToken {
kind: Bogus,
range: 39..40,
},
SimpleToken {
kind: Bogus,
range: 38..39,
},
SimpleToken {
kind: Bogus,
range: 37..38,
},
SimpleToken {
kind: Bogus,
range: 36..37,
},
SimpleToken {
kind: Bogus,
range: 35..36,
},
SimpleToken {
kind: Bogus,
range: 34..35,
},
SimpleToken {
kind: Bogus,
range: 33..34,
},
SimpleToken {
kind: Bogus,
range: 32..33,
},
SimpleToken {
kind: Bogus,
range: 31..32,
},
SimpleToken {
kind: Bogus,
range: 30..31,
},
SimpleToken {
kind: Bogus,
range: 29..30,
},
SimpleToken {
kind: Bogus,
range: 28..29,
},
SimpleToken {
kind: Bogus,
range: 27..28,
},
SimpleToken {
kind: Bogus,
range: 26..27,
},
SimpleToken {
kind: Bogus,
range: 25..26,
},
SimpleToken {
kind: Bogus,
range: 24..25,
},
SimpleToken {
kind: Bogus,
range: 23..24,
},
SimpleToken {
kind: Bogus,
range: 22..23,
},
SimpleToken {
kind: Bogus,
range: 21..22,
},
SimpleToken {
kind: Bogus,
range: 20..21,
},
SimpleToken {
kind: Bogus,
range: 19..20,
},
SimpleToken {
kind: Bogus,
range: 18..19,
},
SimpleToken {
kind: Bogus,
range: 17..18,
},
SimpleToken {
kind: Bogus,
range: 16..17,
},
SimpleToken {
kind: Bogus,
range: 15..16,
},
SimpleToken {
kind: Bogus,
range: 14..15,
},
SimpleToken {
kind: Bogus,
range: 13..14,
},
SimpleToken {
kind: Bogus,
range: 12..13,
},
SimpleToken {
kind: Bogus,
range: 11..12,
},
SimpleToken {
kind: Bogus,
range: 10..11,
},
SimpleToken {
kind: Bogus,
range: 9..10,
},
SimpleToken {
kind: Bogus,
range: 8..9,
},
SimpleToken {
kind: Bogus,
range: 7..8,
},
SimpleToken {
kind: Bogus,
range: 6..7,
},
SimpleToken {
kind: Bogus,
range: 5..6,
},
SimpleToken {
kind: Bogus,
range: 4..5,
},
SimpleToken {
kind: Bogus,
range: 3..4,
},
SimpleToken {
kind: Bogus,
range: 2..3,
},
SimpleToken {
kind: Bogus,
range: 1..2,
},
SimpleToken {
kind: Bogus,
range: 0..1,
range: 0..79,
},
]

12
crates/ruff_python_trivia/src/tokenizer.rs
View file

@ -504,10 +504,12 @@ impl<'a> SimpleTokenizer<'a> {
// Emit a single final bogus token
let token = SimpleToken {
kind: SimpleTokenKind::Bogus,
range: TextRange::at(self.offset, first.text_len()),
range: TextRange::new(self.offset, self.source.text_len()),
};
self.offset += first.text_len();
// Set the cursor to EOF
self.cursor = Cursor::new("");
self.offset = self.source.text_len();
return token;
}
@ -786,10 +788,12 @@ impl<'a> BackwardsTokenizer<'a> {
if self.bogus {
let token = SimpleToken {
kind: SimpleTokenKind::Bogus,
range: TextRange::at(self.back_offset - last.text_len(), last.text_len()),
range: TextRange::up_to(self.back_offset),
};
self.back_offset -= last.text_len();
// Set the cursor to EOF
self.cursor = Cursor::new("");
self.back_offset = TextSize::new(0);
return token;
}