mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-18 05:00:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Support the string concat operator (#178)

Browse source 
The selected precedence is based on BigQuery documentation, where it is equal to `*` and `/`:

https://cloud.google.com/bigquery/docs/reference/standard-sql/operators
This commit is contained in:
Daniël Heres 2020-06-02 20:24:30 +02:00 • committed by GitHub
parent 5f3c1bda01
commit 00dc490f72
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 52 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

33
src/tokenizer.rs
View file

@ -64,6 +64,8 @@ pub enum Token {
Div,
/// Modulo Operator `%`
Mod,
/// String concatenation `||`
StringConcat,
/// Left parenthesis `(`
LParen,
/// Right parenthesis `)`
@ -111,6 +113,7 @@ impl fmt::Display for Token {
Token::Minus => f.write_str("-"),
Token::Mult => f.write_str("*"),
Token::Div => f.write_str("/"),
Token::StringConcat => f.write_str("||"),
Token::Mod => f.write_str("%"),
Token::LParen => f.write_str("("),
Token::RParen => f.write_str(")"),
@ -374,6 +377,16 @@ impl<'a> Tokenizer<'a> {
'+' => self.consume_and_return(chars, Token::Plus),
'*' => self.consume_and_return(chars, Token::Mult),
'%' => self.consume_and_return(chars, Token::Mod),
'|' => {
chars.next(); // consume the '|'
match chars.peek() {
Some('|') => self.consume_and_return(chars, Token::StringConcat),
_ => Err(TokenizerError(format!(
"Expecting to see `||`. Bitwise or operator `|` is not supported. \nError at Line: {}, Col: {}",
self.line, self.col
))),
}
}
'=' => self.consume_and_return(chars, Token::Eq),
'.' => self.consume_and_return(chars, Token::Period),
'!' => {
@ -562,6 +575,26 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_string_string_concat() {
let sql = String::from("SELECT 'a' || 'b'");
let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();
let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::SingleQuotedString(String::from("a")),
Token::Whitespace(Whitespace::Space),
Token::StringConcat,
Token::Whitespace(Whitespace::Space),
Token::SingleQuotedString(String::from("b")),
];
compare(expected, tokens);
}
#[test]
fn tokenize_simple_select() {
let sql = String::from("SELECT * FROM customer WHERE id = 1 LIMIT 5");