mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-10-17 17:27:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Fix the parsing result for the special double number (#1621)

Browse source
This commit is contained in:
Jax Liu 2024-12-28 21:16:30 +08:00 • committed by GitHub
parent 6daa4b059c
commit d0d4153137
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 410 additions and 265 deletions
Download patch file Download diff file Expand all files Collapse all files

45
src/tokenizer.rs
View file

@ -1144,30 +1144,16 @@ impl<'a> Tokenizer<'a> {
// match one period
if let Some('.') = chars.peek() {
// Check if this actually is a float point number
let mut char_clone = chars.peekable.clone();
char_clone.next();
// Next char should be a digit, otherwise, it is not a float point number
if char_clone
.peek()
.map(|c| c.is_ascii_digit())
.unwrap_or(false)
{
s.push('.');
chars.next();
} else if !s.is_empty() {
// Number might be part of period separated construct. Keep the period for next token
// e.g. a-12.b
return Ok(Some(Token::Number(s, false)));
} else {
// No number -> Token::Period
chars.next();
return Ok(Some(Token::Period));
}
s.push('.');
chars.next();
}
s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
// No number -> Token::Period
if s == "." {
return Ok(Some(Token::Period));
}
let mut exponent_part = String::new();
// Parse exponent as number
if chars.peek() == Some(&'e') || chars.peek() == Some(&'E') {
@ -2199,23 +2185,6 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_select_float_hyphenated_identifier() {
let sql = String::from("SELECT a-12.b");
let dialect = GenericDialect {};
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::make_word("a", None),
Token::Minus,
Token::Number(String::from("12"), false),
Token::Period,
Token::make_word("b", None),
];
compare(expected, tokens);
}
#[test]
fn tokenize_clickhouse_double_equal() {
let sql = String::from("SELECT foo=='1'");