Remove duplication around is_trivia functions (#11956)

## Summary

This PR removes the duplication around `is_trivia` functions.

There are two of them in the codebase:
1. In `pycodestyle`, it's for newline, indent, dedent, non-logical
newline and comment
2. In the parser, it's for non-logical newline and comment

The `TokenKind::is_trivia` method used (1) but that's not correct in
that context. So, this PR introduces a new `is_non_logical_token` helper
method for the `pycodestyle` crate and updates the
`TokenKind::is_trivia` implementation with (2).

This also means we can remove `Token::is_trivia` method and the
standalone `token_source::is_trivia` function and use the one on
`TokenKind`.

## Test Plan

`cargo insta test`
This commit is contained in:
Dhruv Manilawala 2024-06-21 15:32:40 +05:30 committed by GitHub
parent 690e94f4fb
commit 4667d8697c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 41 additions and 59 deletions

View file

@ -1626,12 +1626,6 @@ impl Token {
(self.kind, self.range)
}
/// Returns `true` if this is a trivia token.
#[inline]
pub const fn is_trivia(self) -> bool {
matches!(self.kind, TokenKind::Comment | TokenKind::NonLogicalNewline)
}
/// Returns `true` if this is any kind of string token.
const fn is_any_string(self) -> bool {
matches!(

View file

@ -192,13 +192,15 @@ pub enum TokenKind {
}
impl TokenKind {
/// Returns `true` if this is an end of file token.
#[inline]
pub const fn is_eof(self) -> bool {
matches!(self, TokenKind::EndOfFile)
}
/// Returns `true` if this is either a newline or non-logical newline token.
#[inline]
pub const fn is_newline(self) -> bool {
pub const fn is_any_newline(self) -> bool {
matches!(self, TokenKind::Newline | TokenKind::NonLogicalNewline)
}
@ -294,21 +296,16 @@ impl TokenKind {
)
}
/// Returns `true` if this is a singleton token i.e., `True`, `False`, or `None`.
#[inline]
pub const fn is_singleton(self) -> bool {
matches!(self, TokenKind::False | TokenKind::True | TokenKind::None)
}
/// Returns `true` if this is a trivia token i.e., a comment or a non-logical newline.
#[inline]
pub const fn is_trivia(&self) -> bool {
matches!(
self,
TokenKind::Newline
| TokenKind::Indent
| TokenKind::Dedent
| TokenKind::NonLogicalNewline
| TokenKind::Comment
)
matches!(self, TokenKind::Comment | TokenKind::NonLogicalNewline)
}
#[inline]

View file

@ -114,7 +114,7 @@ impl<'src> TokenSource<'src> {
fn do_bump(&mut self) {
loop {
let kind = self.lexer.next_token();
if is_trivia(kind) {
if kind.is_trivia() {
self.tokens
.push(Token::new(kind, self.current_range(), self.current_flags()));
continue;
@ -127,7 +127,7 @@ impl<'src> TokenSource<'src> {
fn next_non_trivia_token(&mut self) -> TokenKind {
loop {
let kind = self.lexer.next_token();
if is_trivia(kind) {
if kind.is_trivia() {
continue;
}
break kind;
@ -187,7 +187,3 @@ fn allocate_tokens_vec(contents: &str) -> Vec<Token> {
let lower_bound = contents.len().saturating_mul(15) / 100;
Vec::with_capacity(lower_bound)
}
fn is_trivia(token: TokenKind) -> bool {
matches!(token, TokenKind::Comment | TokenKind::NonLogicalNewline)
}