mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-18 05:00:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Support bitwise and, or, xor (#181)

Browse source 
Operator precedence is coming from:

https://cloud.google.com/bigquery/docs/reference/standard-sql/operators
This commit is contained in:
Daniël Heres 2020-06-03 18:02:05 +02:00 • committed by GitHub
parent 00dc490f72
commit b4699bd4a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 66 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

38
src/tokenizer.rs
View file

@ -84,8 +84,12 @@ pub enum Token {
LBracket,
/// Right bracket `]`
RBracket,
/// Ampersand &
/// Ampersand `&`
Ampersand,
/// Pipe `|`
Pipe,
/// Caret `^`
Caret,
/// Left brace `{`
LBrace,
/// Right brace `}`
@ -125,6 +129,8 @@ impl fmt::Display for Token {
Token::LBracket => f.write_str("["),
Token::RBracket => f.write_str("]"),
Token::Ampersand => f.write_str("&"),
Token::Caret => f.write_str("^"),
Token::Pipe => f.write_str("|"),
Token::LBrace => f.write_str("{"),
Token::RBrace => f.write_str("}"),
}
@ -381,10 +387,8 @@ impl<'a> Tokenizer<'a> {
chars.next(); // consume the '|'
match chars.peek() {
Some('|') => self.consume_and_return(chars, Token::StringConcat),
_ => Err(TokenizerError(format!(
"Expecting to see `||`. Bitwise or operator `|` is not supported. \nError at Line: {}, Col: {}",
self.line, self.col
))),
// Bitshift '|' operator
_ => Ok(Some(Token::Pipe)),
}
}
'=' => self.consume_and_return(chars, Token::Eq),
@ -426,6 +430,7 @@ impl<'a> Tokenizer<'a> {
'[' => self.consume_and_return(chars, Token::LBracket),
']' => self.consume_and_return(chars, Token::RBracket),
'&' => self.consume_and_return(chars, Token::Ampersand),
'^' => self.consume_and_return(chars, Token::Caret),
'{' => self.consume_and_return(chars, Token::LBrace),
'}' => self.consume_and_return(chars, Token::RBrace),
other => self.consume_and_return(chars, Token::Char(other)),
@ -594,6 +599,29 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_bitwise_op() {
let sql = String::from("SELECT one | two ^ three");
let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();
let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::make_word("one", None),
Token::Whitespace(Whitespace::Space),
Token::Pipe,
Token::Whitespace(Whitespace::Space),
Token::make_word("two", None),
Token::Whitespace(Whitespace::Space),
Token::Caret,
Token::Whitespace(Whitespace::Space),
Token::make_word("three", None),
];
compare(expected, tokens);
}
#[test]
fn tokenize_simple_select() {