mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-09 22:25:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve CharWindow

Browse source
This commit is contained in:
harupy 2023-01-03 17:27:35 +09:00
parent b6647b0171
commit 0b65609d26
1 changed files with 22 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

47
parser/src/lexer.rs
View file

@ -124,12 +124,12 @@ where
impl<T, const N: usize, Idx> Index<Idx> for CharWindow<T, N> impl<T, const N: usize, Idx> Index<Idx> for CharWindow<T, N>
where where
T: Iterator<Item = char>, T: Iterator<Item = char>,
Idx: SliceIndex<[Option<char>], Output = Option<char>>, Idx: SliceIndex<[Option<char>]>,
{ {
type Output = Option<char>; type Output = Idx::Output;
fn index(&self, index: Idx) -> &Self::Output { fn index(&self, index: Idx) -> &Self::Output {
self.window.index(index) &self.window[index]
} }
} }
@ -199,12 +199,12 @@ where
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
// Collapse \r\n into \n // Collapse \r\n into \n
loop { loop {
match (self.window[0], self.window[1]) { match self.window[..2] {
(Some('\r'), Some('\n')) => { [Some('\r'), Some('\n')] => {
// Windows EOL into \n // Windows EOL into \n
self.shift(); self.shift();
} }
(Some('\r'), _) => { [Some('\r'), _] => {
// MAC EOL into \n // MAC EOL into \n
self.window.change_first('\n'); self.window.change_first('\n');
} }
@ -309,20 +309,20 @@ where
/// Numeric lexing. The feast can start! /// Numeric lexing. The feast can start!
fn lex_number(&mut self) -> LexResult { fn lex_number(&mut self) -> LexResult {
let start_pos = self.get_pos(); let start_pos = self.get_pos();
match (self.window[0], self.window[1]) { match self.window[..2] {
(Some('0'), Some('x' | 'X')) => { [Some('0'), Some('x' | 'X')] => {
// Hex! (0xdeadbeef) // Hex! (0xdeadbeef)
self.next_char(); self.next_char();
self.next_char(); self.next_char();
self.lex_number_radix(start_pos, 16) self.lex_number_radix(start_pos, 16)
} }
(Some('0'), Some('o' | 'O')) => { [Some('0'), Some('o' | 'O')] => {
// Octal style! (0o377) // Octal style! (0o377)
self.next_char(); self.next_char();
self.next_char(); self.next_char();
self.lex_number_radix(start_pos, 8) self.lex_number_radix(start_pos, 8)
} }
(Some('0'), Some('b' | 'B')) => { [Some('0'), Some('b' | 'B')] => {
// Binary! (0b_1110_0101) // Binary! (0b_1110_0101)
self.next_char(); self.next_char();
self.next_char(); self.next_char();
@ -470,9 +470,9 @@ where
/// Test if we face '[eE][-+]?[0-9]+' /// Test if we face '[eE][-+]?[0-9]+'
fn at_exponent(&self) -> bool { fn at_exponent(&self) -> bool {
match (self.window[0], self.window[1]) { match self.window[..2] {
(Some('e' | 'E'), Some('+' | '-')) => matches!(self.window[2], Some('0'..='9')), [Some('e' | 'E'), Some('+' | '-')] => matches!(self.window[2], Some('0'..='9')),
(Some('e' | 'E'), Some('0'..='9')) => true, [Some('e' | 'E'), Some('0'..='9')] => true,
_ => false, _ => false,
} }
} }
@ -500,14 +500,13 @@ where
// If the next two characters are also the quote character, then we have a triple-quoted // If the next two characters are also the quote character, then we have a triple-quoted
// string; consume those two characters and ensure that we require a triple-quote to close // string; consume those two characters and ensure that we require a triple-quote to close
let triple_quoted = let triple_quoted = if self.window[..2] == [Some(quote_char); 2] {
if self.window[0] == Some(quote_char) && self.window[1] == Some(quote_char) { self.next_char();
self.next_char(); self.next_char();
self.next_char(); true
true } else {
} else { false
false };
};
loop { loop {
match self.next_char() { match self.next_char() {
@ -534,9 +533,7 @@ where
// Look ahead at the next two characters; if we have two more // Look ahead at the next two characters; if we have two more
// quote_chars, it's the end of the string; consume the remaining // quote_chars, it's the end of the string; consume the remaining
// closing quotes and break the loop // closing quotes and break the loop
if self.window[0] == Some(quote_char) if self.window[..2] == [Some(quote_char); 2] {
&& self.window[1] == Some(quote_char)
{
self.next_char(); self.next_char();
self.next_char(); self.next_char();
break; break;
@ -1094,7 +1091,7 @@ where
} else { } else {
let tok_start = self.get_pos(); let tok_start = self.get_pos();
self.next_char(); self.next_char();
if let (Some('.'), Some('.')) = (&self.window[0], &self.window[1]) { if let [Some('.'), Some('.')] = self.window[..2] {
self.next_char(); self.next_char();
self.next_char(); self.next_char();
let tok_end = self.get_pos(); let tok_end = self.get_pos();