mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-31 11:17:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Reduce duplication in tokenizer by matching on Some('...') directly

Browse source
This commit is contained in:
Nickolay Ponomarev 2019-06-08 04:15:20 +03:00
parent 5536cd1f9e
commit ebc5efda98
1 changed files with 9 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

36
src/sqltokenizer.rs
View file

@ -394,14 +394,8 @@ impl<'a> Tokenizer<'a> {
'!' => {
chars.next(); // consume
match chars.peek() {
Some(&ch) => match ch {
'=' => self.consume_and_return(chars, Token::Neq),
_ => Err(TokenizerError(format!(
"Tokenizer Error at Line: {}, Col: {}",
self.line, self.col
))),
},
None => Err(TokenizerError(format!(
Some('=') => self.consume_and_return(chars, Token::Neq),
_ => Err(TokenizerError(format!(
"Tokenizer Error at Line: {}, Col: {}",
self.line, self.col
))),
@ -410,39 +404,27 @@ impl<'a> Tokenizer<'a> {
'<' => {
chars.next(); // consume
match chars.peek() {
Some(&ch) => match ch {
'=' => self.consume_and_return(chars, Token::LtEq),
'>' => self.consume_and_return(chars, Token::Neq),
_ => Ok(Some(Token::Lt)),
},
None => Ok(Some(Token::Lt)),
Some('=') => self.consume_and_return(chars, Token::LtEq),
Some('>') => self.consume_and_return(chars, Token::Neq),
_ => Ok(Some(Token::Lt)),
}
}
'>' => {
chars.next(); // consume
match chars.peek() {
Some(&ch) => match ch {
'=' => self.consume_and_return(chars, Token::GtEq),
_ => Ok(Some(Token::Gt)),
},
None => Ok(Some(Token::Gt)),
Some('=') => self.consume_and_return(chars, Token::GtEq),
_ => Ok(Some(Token::Gt)),
}
}
// colon
':' => {
chars.next();
match chars.peek() {
Some(&ch) => match ch {
// double colon
':' => self.consume_and_return(chars, Token::DoubleColon),
_ => Ok(Some(Token::Colon)),
},
None => Ok(Some(Token::Colon)),
Some(':') => self.consume_and_return(chars, Token::DoubleColon),
_ => Ok(Some(Token::Colon)),
}
}
';' => self.consume_and_return(chars, Token::SemiColon),
'\\' => self.consume_and_return(chars, Token::Backslash),
// brakets
'[' => self.consume_and_return(chars, Token::LBracket),
']' => self.consume_and_return(chars, Token::RBracket),
'&' => self.consume_and_return(chars, Token::Ampersand),