mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-10-16 08:49:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Fix BigQuery hyphenated ObjectName with numbers (#1598)

Browse source
This commit is contained in:
Ayman Elkfrawy 2024-12-18 12:12:09 -08:00 • committed by GitHub
parent 8fcdf48e5c
commit e9ab4d6b94
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 61 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/tokenizer.rs
View file

@ -1144,15 +1144,29 @@ impl<'a> Tokenizer<'a> {
// match one period
if let Some('.') = chars.peek() {
s.push('.');
chars.next();
// Check if this actually is a float point number
let mut char_clone = chars.peekable.clone();
char_clone.next();
// Next char should be a digit, otherwise, it is not a float point number
if char_clone
.peek()
.map(|c| c.is_ascii_digit())
.unwrap_or(false)
{
s.push('.');
chars.next();
} else if !s.is_empty() {
// Number might be part of period separated construct. Keep the period for next token
// e.g. a-12.b
return Ok(Some(Token::Number(s, false)));
} else {
// No number -> Token::Period
chars.next();
return Ok(Some(Token::Period));
}
}
s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
// No number -> Token::Period
if s == "." {
return Ok(Some(Token::Period));
}
s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
let mut exponent_part = String::new();
// Parse exponent as number
@ -2185,6 +2199,23 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_select_float_hyphenated_identifier() {
let sql = String::from("SELECT a-12.b");
let dialect = GenericDialect {};
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::make_word("a", None),
Token::Minus,
Token::Number(String::from("12"), false),
Token::Period,
Token::make_word("b", None),
];
compare(expected, tokens);
}
#[test]
fn tokenize_clickhouse_double_equal() {
let sql = String::from("SELECT foo=='1'");