mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-08 05:35:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Don't call is_emoji_presentation for each invocation of consume_normal

Browse source
This commit is contained in:
Dimitris Fasarakis Hilliard 2023-01-29 20:08:21 +02:00
parent f4831432f9
commit 2a4d44d667
1 changed files with 24 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

48
parser/src/lexer.rs
View file

@ -689,24 +689,12 @@ where
fn consume_normal(&mut self) -> Result<(), LexicalError> {
// Check if we have some character:
if let Some(c) = self.window[0] {
// First check identifier:
if self.is_identifier_start(c) {
let identifier = self.lex_identifier()?;
self.emit(identifier);
} else if is_emoji_presentation(c) {
let tok_start = self.get_pos();
self.next_char();
let tok_end = self.get_pos();
self.emit((
tok_start,
Tok::Name {
name: c.to_string(),
},
tok_end,
));
} else {
self.consume_character(c)?;
}
if self.is_identifier_start(c) {
let identifier = self.lex_identifier()?;
self.emit(identifier);
} else {
self.consume_character(c)?;
}
} else {
// We reached end of file.
let tok_pos = self.get_pos();
@ -1112,13 +1100,25 @@ where
});
}
}
_ => {
let c = self.next_char();
return Err(LexicalError {
error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
location: self.get_pos(),
});
if is_emoji_presentation(c) {
let tok_start = self.get_pos();
self.next_char();
let tok_end = self.get_pos();
self.emit((
tok_start,
Tok::Name {
name: c.to_string(),
},
tok_end,
));
} else {
let c = self.next_char();
return Err(LexicalError {
error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
location: self.get_pos(),
});
}
} // Ignore all the rest..
}