diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap index 970bcd7e1c..98b020e6e8 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__Reverse.snap @@ -13,206 +13,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 50..51, - }, - SimpleToken { - kind: Bogus, - range: 49..50, - }, - SimpleToken { - kind: Bogus, - range: 48..49, - }, - SimpleToken { - kind: Bogus, - range: 47..48, - }, - SimpleToken { - kind: Bogus, - range: 46..47, - }, - SimpleToken { - kind: Bogus, - range: 45..46, - }, - SimpleToken { - kind: Bogus, - range: 44..45, - }, - SimpleToken { - kind: Bogus, - range: 43..44, - }, - SimpleToken { - kind: Bogus, - range: 42..43, - }, - SimpleToken { - kind: Bogus, - range: 41..42, - }, - SimpleToken { - kind: Bogus, - range: 40..41, - }, - SimpleToken { - kind: Bogus, - range: 39..40, - }, - SimpleToken { - kind: Bogus, - range: 38..39, - }, - SimpleToken { - kind: Bogus, - range: 37..38, - }, - SimpleToken { - kind: Bogus, - range: 36..37, - }, - SimpleToken { - kind: Bogus, - range: 35..36, - }, - SimpleToken { - kind: Bogus, - range: 34..35, - }, - SimpleToken { - kind: Bogus, - range: 33..34, - }, - SimpleToken { - kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 26..27, - }, - SimpleToken { - kind: Bogus, - range: 25..26, - }, - SimpleToken { - kind: Bogus, - range: 24..25, - }, - SimpleToken { - kind: Bogus, - range: 23..24, - }, - SimpleToken { - kind: Bogus, - range: 22..23, - }, - SimpleToken { - kind: Bogus, - range: 21..22, - }, - SimpleToken { - kind: Bogus, - range: 20..21, - }, - SimpleToken { - kind: Bogus, - range: 19..20, - }, - SimpleToken { - kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..51, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap index 1afba81fcc..1ebd8a5f48 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_single_quoted_string.snap @@ -17,62 +17,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..15, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap index a3585ac5e4..f2958e6691 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__comment_containing_triple_quoted_string.snap @@ -17,78 +17,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..19, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap index a9c0c43214..24b9b16ccc 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__ignore_word_with_only_id_continuing_chars.snap @@ -9,10 +9,6 @@ expression: test_case.tokens() }, SimpleToken { kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 2..3, + range: 1..3, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap index 60b5a8da36..5836fce453 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_containing_comment.snap @@ -9,306 +9,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 75..76, - }, - SimpleToken { - kind: Bogus, - range: 74..75, - }, - SimpleToken { - kind: Bogus, - range: 73..74, - }, - SimpleToken { - kind: Bogus, - range: 72..73, - }, - SimpleToken { - kind: Bogus, - range: 71..72, - }, - SimpleToken { - kind: Bogus, - range: 70..71, - }, - SimpleToken { - kind: Bogus, - range: 69..70, - }, - SimpleToken { - kind: Bogus, - range: 68..69, - }, - SimpleToken { - kind: Bogus, - range: 67..68, - }, - SimpleToken { - kind: Bogus, - range: 66..67, - }, - SimpleToken { - kind: Bogus, - range: 65..66, - }, - SimpleToken { - kind: Bogus, - range: 64..65, - }, - SimpleToken { - kind: Bogus, - range: 63..64, - }, - SimpleToken { - kind: Bogus, - range: 62..63, - }, - SimpleToken { - kind: Bogus, - range: 61..62, - }, - SimpleToken { - kind: Bogus, - range: 60..61, - }, - SimpleToken { - kind: Bogus, - range: 59..60, - }, - SimpleToken { - kind: Bogus, - range: 58..59, - }, - SimpleToken { - kind: Bogus, - range: 57..58, - }, - SimpleToken { - kind: Bogus, - range: 56..57, - }, - SimpleToken { - kind: Bogus, - range: 55..56, - }, - SimpleToken { - kind: Bogus, - range: 54..55, - }, - SimpleToken { - kind: Bogus, - range: 53..54, - }, - SimpleToken { - kind: Bogus, - range: 52..53, - }, - SimpleToken { - kind: Bogus, - range: 51..52, - }, - SimpleToken { - kind: Bogus, - range: 50..51, - }, - SimpleToken { - kind: Bogus, - range: 49..50, - }, - SimpleToken { - kind: Bogus, - range: 48..49, - }, - SimpleToken { - kind: Bogus, - range: 47..48, - }, - SimpleToken { - kind: Bogus, - range: 46..47, - }, - SimpleToken { - kind: Bogus, - range: 45..46, - }, - SimpleToken { - kind: Bogus, - range: 44..45, - }, - SimpleToken { - kind: Bogus, - range: 43..44, - }, - SimpleToken { - kind: Bogus, - range: 42..43, - }, - SimpleToken { - kind: Bogus, - range: 41..42, - }, - SimpleToken { - kind: Bogus, - range: 40..41, - }, - SimpleToken { - kind: Bogus, - range: 39..40, - }, - SimpleToken { - kind: Bogus, - range: 38..39, - }, - SimpleToken { - kind: Bogus, - range: 37..38, - }, - SimpleToken { - kind: Bogus, - range: 36..37, - }, - SimpleToken { - kind: Bogus, - range: 35..36, - }, - SimpleToken { - kind: Bogus, - range: 34..35, - }, - SimpleToken { - kind: Bogus, - range: 33..34, - }, - SimpleToken { - kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 26..27, - }, - SimpleToken { - kind: Bogus, - range: 25..26, - }, - SimpleToken { - kind: Bogus, - range: 24..25, - }, - SimpleToken { - kind: Bogus, - range: 23..24, - }, - SimpleToken { - kind: Bogus, - range: 22..23, - }, - SimpleToken { - kind: Bogus, - range: 21..22, - }, - SimpleToken { - kind: Bogus, - range: 20..21, - }, - SimpleToken { - kind: Bogus, - range: 19..20, - }, - SimpleToken { - kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..76, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap index ee103aeece..774475fc23 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__single_quoted_multiline_string_implicit_concatenation.snap @@ -9,314 +9,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 77..78, - }, - SimpleToken { - kind: Bogus, - range: 76..77, - }, - SimpleToken { - kind: Bogus, - range: 75..76, - }, - SimpleToken { - kind: Bogus, - range: 74..75, - }, - SimpleToken { - kind: Bogus, - range: 73..74, - }, - SimpleToken { - kind: Bogus, - range: 72..73, - }, - SimpleToken { - kind: Bogus, - range: 71..72, - }, - SimpleToken { - kind: Bogus, - range: 70..71, - }, - SimpleToken { - kind: Bogus, - range: 69..70, - }, - SimpleToken { - kind: Bogus, - range: 68..69, - }, - SimpleToken { - kind: Bogus, - range: 67..68, - }, - SimpleToken { - kind: Bogus, - range: 66..67, - }, - SimpleToken { - kind: Bogus, - range: 65..66, - }, - SimpleToken { - kind: Bogus, - range: 64..65, - }, - SimpleToken { - kind: Bogus, - range: 63..64, - }, - SimpleToken { - kind: Bogus, - range: 62..63, - }, - SimpleToken { - kind: Bogus, - range: 61..62, - }, - SimpleToken { - kind: Bogus, - range: 60..61, - }, - SimpleToken { - kind: Bogus, - range: 59..60, - }, - SimpleToken { - kind: Bogus, - range: 58..59, - }, - SimpleToken { - kind: Bogus, - range: 57..58, - }, - SimpleToken { - kind: Bogus, - range: 56..57, - }, - SimpleToken { - kind: Bogus, - range: 55..56, - }, - SimpleToken { - kind: Bogus, - range: 54..55, - }, - SimpleToken { - kind: Bogus, - range: 53..54, - }, - SimpleToken { - kind: Bogus, - range: 52..53, - }, - SimpleToken { - kind: Bogus, - range: 51..52, - }, - SimpleToken { - kind: Bogus, - range: 50..51, - }, - SimpleToken { - kind: Bogus, - range: 49..50, - }, - SimpleToken { - kind: Bogus, - range: 48..49, - }, - SimpleToken { - kind: Bogus, - range: 47..48, - }, - SimpleToken { - kind: Bogus, - range: 46..47, - }, - SimpleToken { - kind: Bogus, - range: 45..46, - }, - SimpleToken { - kind: Bogus, - range: 44..45, - }, - SimpleToken { - kind: Bogus, - range: 43..44, - }, - SimpleToken { - kind: Bogus, - range: 42..43, - }, - SimpleToken { - kind: Bogus, - range: 41..42, - }, - SimpleToken { - kind: Bogus, - range: 40..41, - }, - SimpleToken { - kind: Bogus, - range: 39..40, - }, - SimpleToken { - kind: Bogus, - range: 38..39, - }, - SimpleToken { - kind: Bogus, - range: 37..38, - }, - SimpleToken { - kind: Bogus, - range: 36..37, - }, - SimpleToken { - kind: Bogus, - range: 35..36, - }, - SimpleToken { - kind: Bogus, - range: 34..35, - }, - SimpleToken { - kind: Bogus, - range: 33..34, - }, - SimpleToken { - kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 26..27, - }, - SimpleToken { - kind: Bogus, - range: 25..26, - }, - SimpleToken { - kind: Bogus, - range: 24..25, - }, - SimpleToken { - kind: Bogus, - range: 23..24, - }, - SimpleToken { - kind: Bogus, - range: 22..23, - }, - SimpleToken { - kind: Bogus, - range: 21..22, - }, - SimpleToken { - kind: Bogus, - range: 20..21, - }, - SimpleToken { - kind: Bogus, - range: 19..20, - }, - SimpleToken { - kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..78, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap index 7e2f64da75..01da094943 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_followed_by_multiple_comments.snap @@ -17,206 +17,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 50..51, - }, - SimpleToken { - kind: Bogus, - range: 49..50, - }, - SimpleToken { - kind: Bogus, - range: 48..49, - }, - SimpleToken { - kind: Bogus, - range: 47..48, - }, - SimpleToken { - kind: Bogus, - range: 46..47, - }, - SimpleToken { - kind: Bogus, - range: 45..46, - }, - SimpleToken { - kind: Bogus, - range: 44..45, - }, - SimpleToken { - kind: Bogus, - range: 43..44, - }, - SimpleToken { - kind: Bogus, - range: 42..43, - }, - SimpleToken { - kind: Bogus, - range: 41..42, - }, - SimpleToken { - kind: Bogus, - range: 40..41, - }, - SimpleToken { - kind: Bogus, - range: 39..40, - }, - SimpleToken { - kind: Bogus, - range: 38..39, - }, - SimpleToken { - kind: Bogus, - range: 37..38, - }, - SimpleToken { - kind: Bogus, - range: 36..37, - }, - SimpleToken { - kind: Bogus, - range: 35..36, - }, - SimpleToken { - kind: Bogus, - range: 34..35, - }, - SimpleToken { - kind: Bogus, - range: 33..34, - }, - SimpleToken { - kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 26..27, - }, - SimpleToken { - kind: Bogus, - range: 25..26, - }, - SimpleToken { - kind: Bogus, - range: 24..25, - }, - SimpleToken { - kind: Bogus, - range: 23..24, - }, - SimpleToken { - kind: Bogus, - range: 22..23, - }, - SimpleToken { - kind: Bogus, - range: 21..22, - }, - SimpleToken { - kind: Bogus, - range: 20..21, - }, - SimpleToken { - kind: Bogus, - range: 19..20, - }, - SimpleToken { - kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..51, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap index 756d0e7075..ce46e6154a 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_double_escaped_backslash.snap @@ -17,50 +17,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..12, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap index b9e074fbb3..7e1735c3ed 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__string_with_escaped_quote.snap @@ -17,134 +17,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 26..27, - }, - SimpleToken { - kind: Bogus, - range: 25..26, - }, - SimpleToken { - kind: Bogus, - range: 24..25, - }, - SimpleToken { - kind: Bogus, - range: 23..24, - }, - SimpleToken { - kind: Bogus, - range: 22..23, - }, - SimpleToken { - kind: Bogus, - range: 21..22, - }, - SimpleToken { - kind: Bogus, - range: 20..21, - }, - SimpleToken { - kind: Bogus, - range: 19..20, - }, - SimpleToken { - kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..33, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap index f2678780c0..1b2a383429 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__tokenize_bogus.snap @@ -21,106 +21,6 @@ expression: test_case.tokens() }, SimpleToken { kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 33..34, - }, - SimpleToken { - kind: Bogus, - range: 34..35, - }, - SimpleToken { - kind: Bogus, - range: 35..36, - }, - SimpleToken { - kind: Bogus, - range: 36..37, - }, - SimpleToken { - kind: Bogus, - range: 37..38, - }, - SimpleToken { - kind: Bogus, - range: 38..39, - }, - SimpleToken { - kind: Bogus, - range: 39..40, - }, - SimpleToken { - kind: Bogus, - range: 40..41, - }, - SimpleToken { - kind: Bogus, - range: 41..42, - }, - SimpleToken { - kind: Bogus, - range: 42..43, - }, - SimpleToken { - kind: Bogus, - range: 43..44, - }, - SimpleToken { - kind: Bogus, - range: 44..45, - }, - SimpleToken { - kind: Bogus, - range: 45..46, - }, - SimpleToken { - kind: Bogus, - range: 46..47, - }, - SimpleToken { - kind: Bogus, - range: 47..48, - }, - SimpleToken { - kind: Bogus, - range: 48..49, - }, - SimpleToken { - kind: Bogus, - range: 49..50, - }, - SimpleToken { - kind: Bogus, - range: 50..51, - }, - SimpleToken { - kind: Bogus, - range: 51..52, - }, - SimpleToken { - kind: Bogus, - range: 52..53, + range: 27..53, }, ] diff --git a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap index abad16a47d..2708eac19e 100644 --- a/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap +++ b/crates/ruff_python_trivia/src/snapshots/ruff_python_trivia__tokenizer__tests__triple_quoted_multiline_string_containing_comment.snap @@ -9,318 +9,6 @@ expression: test_case.tokenize_reverse() }, SimpleToken { kind: Bogus, - range: 78..79, - }, - SimpleToken { - kind: Bogus, - range: 77..78, - }, - SimpleToken { - kind: Bogus, - range: 76..77, - }, - SimpleToken { - kind: Bogus, - range: 75..76, - }, - SimpleToken { - kind: Bogus, - range: 74..75, - }, - SimpleToken { - kind: Bogus, - range: 73..74, - }, - SimpleToken { - kind: Bogus, - range: 72..73, - }, - SimpleToken { - kind: Bogus, - range: 71..72, - }, - SimpleToken { - kind: Bogus, - range: 70..71, - }, - SimpleToken { - kind: Bogus, - range: 69..70, - }, - SimpleToken { - kind: Bogus, - range: 68..69, - }, - SimpleToken { - kind: Bogus, - range: 67..68, - }, - SimpleToken { - kind: Bogus, - range: 66..67, - }, - SimpleToken { - kind: Bogus, - range: 65..66, - }, - SimpleToken { - kind: Bogus, - range: 64..65, - }, - SimpleToken { - kind: Bogus, - range: 63..64, - }, - SimpleToken { - kind: Bogus, - range: 62..63, - }, - SimpleToken { - kind: Bogus, - range: 61..62, - }, - SimpleToken { - kind: Bogus, - range: 60..61, - }, - SimpleToken { - kind: Bogus, - range: 59..60, - }, - SimpleToken { - kind: Bogus, - range: 58..59, - }, - SimpleToken { - kind: Bogus, - range: 57..58, - }, - SimpleToken { - kind: Bogus, - range: 56..57, - }, - SimpleToken { - kind: Bogus, - range: 55..56, - }, - SimpleToken { - kind: Bogus, - range: 54..55, - }, - SimpleToken { - kind: Bogus, - range: 53..54, - }, - SimpleToken { - kind: Bogus, - range: 52..53, - }, - SimpleToken { - kind: Bogus, - range: 51..52, - }, - SimpleToken { - kind: Bogus, - range: 50..51, - }, - SimpleToken { - kind: Bogus, - range: 49..50, - }, - SimpleToken { - kind: Bogus, - range: 48..49, - }, - SimpleToken { - kind: Bogus, - range: 47..48, - }, - SimpleToken { - kind: Bogus, - range: 46..47, - }, - SimpleToken { - kind: Bogus, - range: 45..46, - }, - SimpleToken { - kind: Bogus, - range: 44..45, - }, - SimpleToken { - kind: Bogus, - range: 43..44, - }, - SimpleToken { - kind: Bogus, - range: 42..43, - }, - SimpleToken { - kind: Bogus, - range: 41..42, - }, - SimpleToken { - kind: Bogus, - range: 40..41, - }, - SimpleToken { - kind: Bogus, - range: 39..40, - }, - SimpleToken { - kind: Bogus, - range: 38..39, - }, - SimpleToken { - kind: Bogus, - range: 37..38, - }, - SimpleToken { - kind: Bogus, - range: 36..37, - }, - SimpleToken { - kind: Bogus, - range: 35..36, - }, - SimpleToken { - kind: Bogus, - range: 34..35, - }, - SimpleToken { - kind: Bogus, - range: 33..34, - }, - SimpleToken { - kind: Bogus, - range: 32..33, - }, - SimpleToken { - kind: Bogus, - range: 31..32, - }, - SimpleToken { - kind: Bogus, - range: 30..31, - }, - SimpleToken { - kind: Bogus, - range: 29..30, - }, - SimpleToken { - kind: Bogus, - range: 28..29, - }, - SimpleToken { - kind: Bogus, - range: 27..28, - }, - SimpleToken { - kind: Bogus, - range: 26..27, - }, - SimpleToken { - kind: Bogus, - range: 25..26, - }, - SimpleToken { - kind: Bogus, - range: 24..25, - }, - SimpleToken { - kind: Bogus, - range: 23..24, - }, - SimpleToken { - kind: Bogus, - range: 22..23, - }, - SimpleToken { - kind: Bogus, - range: 21..22, - }, - SimpleToken { - kind: Bogus, - range: 20..21, - }, - SimpleToken { - kind: Bogus, - range: 19..20, - }, - SimpleToken { - kind: Bogus, - range: 18..19, - }, - SimpleToken { - kind: Bogus, - range: 17..18, - }, - SimpleToken { - kind: Bogus, - range: 16..17, - }, - SimpleToken { - kind: Bogus, - range: 15..16, - }, - SimpleToken { - kind: Bogus, - range: 14..15, - }, - SimpleToken { - kind: Bogus, - range: 13..14, - }, - SimpleToken { - kind: Bogus, - range: 12..13, - }, - SimpleToken { - kind: Bogus, - range: 11..12, - }, - SimpleToken { - kind: Bogus, - range: 10..11, - }, - SimpleToken { - kind: Bogus, - range: 9..10, - }, - SimpleToken { - kind: Bogus, - range: 8..9, - }, - SimpleToken { - kind: Bogus, - range: 7..8, - }, - SimpleToken { - kind: Bogus, - range: 6..7, - }, - SimpleToken { - kind: Bogus, - range: 5..6, - }, - SimpleToken { - kind: Bogus, - range: 4..5, - }, - SimpleToken { - kind: Bogus, - range: 3..4, - }, - SimpleToken { - kind: Bogus, - range: 2..3, - }, - SimpleToken { - kind: Bogus, - range: 1..2, - }, - SimpleToken { - kind: Bogus, - range: 0..1, + range: 0..79, }, ] diff --git a/crates/ruff_python_trivia/src/tokenizer.rs b/crates/ruff_python_trivia/src/tokenizer.rs index a58434ba1f..060c673fc0 100644 --- a/crates/ruff_python_trivia/src/tokenizer.rs +++ b/crates/ruff_python_trivia/src/tokenizer.rs @@ -504,10 +504,12 @@ impl<'a> SimpleTokenizer<'a> { // Emit a single final bogus token let token = SimpleToken { kind: SimpleTokenKind::Bogus, - range: TextRange::at(self.offset, first.text_len()), + range: TextRange::new(self.offset, self.source.text_len()), }; - self.offset += first.text_len(); + // Set the cursor to EOF + self.cursor = Cursor::new(""); + self.offset = self.source.text_len(); return token; } @@ -786,10 +788,12 @@ impl<'a> BackwardsTokenizer<'a> { if self.bogus { let token = SimpleToken { kind: SimpleTokenKind::Bogus, - range: TextRange::at(self.back_offset - last.text_len(), last.text_len()), + range: TextRange::up_to(self.back_offset), }; - self.back_offset -= last.text_len(); + // Set the cursor to EOF + self.cursor = Cursor::new(""); + self.back_offset = TextSize::new(0); return token; }