mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 10:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Fast path for ASCII only identifiers start (#6609)

Browse source
This commit is contained in:
Micha Reiser 2023-08-16 10:22:44 +02:00 • committed by GitHub
parent 2d86e78bfc
commit e28858bb29
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 21 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

10
crates/ruff_python_trivia/src/tokenizer.rs
View file

@ -85,7 +85,11 @@ pub fn lines_after_ignoring_trivia(offset: TextSize, code: &str) -> u32 {
}
fn is_identifier_start(c: char) -> bool {
c.is_ascii_alphabetic() || c == '_' || is_non_ascii_identifier_start(c)
if c.is_ascii() {
c.is_ascii_alphabetic() || c == '_'
} else {
is_xid_start(c)
}
}
// Checks if the character c is a valid continuation character as described
@ -98,10 +102,6 @@ fn is_identifier_continuation(c: char) -> bool {
}
}
fn is_non_ascii_identifier_start(c: char) -> bool {
is_xid_start(c)
}
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub struct SimpleToken {
pub kind: SimpleTokenKind,