mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-30 07:08:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Don't call is_emoji_presentation for each invocation of consume_normal

Browse source
This commit is contained in:
Dimitris Fasarakis Hilliard 2023-01-29 20:08:21 +02:00
parent aa0290bbfc
commit 838990ae15
1 changed files with 24 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

48
parser/src/lexer.rs
View file

@ -689,24 +689,12 @@ where
fn consume_normal(&mut self) -> Result<(), LexicalError> { fn consume_normal(&mut self) -> Result<(), LexicalError> {
// Check if we have some character: // Check if we have some character:
if let Some(c) = self.window[0] { if let Some(c) = self.window[0] {
// First check identifier: if self.is_identifier_start(c) {
if self.is_identifier_start(c) { let identifier = self.lex_identifier()?;
let identifier = self.lex_identifier()?; self.emit(identifier);
self.emit(identifier); } else {
} else if is_emoji_presentation(c) { self.consume_character(c)?;
let tok_start = self.get_pos(); }
self.next_char();
let tok_end = self.get_pos();
self.emit((
tok_start,
Tok::Name {
name: c.to_string(),
},
tok_end,
));
} else {
self.consume_character(c)?;
}
} else { } else {
// We reached end of file. // We reached end of file.
let tok_pos = self.get_pos(); let tok_pos = self.get_pos();
@ -1112,13 +1100,25 @@ where
}); });
} }
} }
_ => { _ => {
let c = self.next_char(); if is_emoji_presentation(c) {
return Err(LexicalError { let tok_start = self.get_pos();
error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() }, self.next_char();
location: self.get_pos(), let tok_end = self.get_pos();
}); self.emit((
tok_start,
Tok::Name {
name: c.to_string(),
},
tok_end,
));
} else {
let c = self.next_char();
return Err(LexicalError {
error: LexicalErrorType::UnrecognizedToken { tok: c.unwrap() },
location: self.get_pos(),
});
}
} // Ignore all the rest.. } // Ignore all the rest..
} }