mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-16 20:20:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Support parsing scientific notation (such as 10e5) (#768)

Browse source
This commit is contained in:
Jeffrey 2022-12-29 00:28:53 +11:00 • committed by GitHub
parent 2d801c9fb6
commit 2c20ec0be5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 118 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

66
src/tokenizer.rs
View file

@ -541,6 +541,7 @@ impl<'a> Tokenizer<'a> {
chars.next(); // consume the first char
let s = self.tokenize_word(ch, chars);
// TODO: implement parsing of exponent here
if s.chars().all(|x| ('0'..='9').contains(&x) || x == '.') {
let mut inner_state = State {
peekable: s.chars().peekable(),
@ -617,6 +618,36 @@ impl<'a> Tokenizer<'a> {
return Ok(Some(Token::Period));
}
// Parse exponent as number
if chars.peek() == Some(&'e') || chars.peek() == Some(&'E') {
let mut char_clone = chars.peekable.clone();
let mut exponent_part = String::new();
exponent_part.push(char_clone.next().unwrap());
// Optional sign
match char_clone.peek() {
Some(&c) if matches!(c, '+' | '-') => {
exponent_part.push(c);
char_clone.next();
}
_ => (),
}
match char_clone.peek() {
// Definitely an exponent, get original iterator up to speed and use it
Some(&c) if matches!(c, '0'..='9') => {
for _ in 0..exponent_part.len() {
chars.next();
}
exponent_part +=
&peeking_take_while(chars, |ch| matches!(ch, '0'..='9'));
s += exponent_part.as_str();
}
// Not an exponent, discard the work done
_ => (),
}
}
let long = if chars.peek() == Some(&'L') {
chars.next();
true
@ -1091,6 +1122,41 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_select_exponent() {
let sql = String::from("SELECT 1e10, 1e-10, 1e+10, 1ea, 1e-10a, 1e-10-10");
let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();
let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::Number(String::from("1e10"), false),
Token::Comma,
Token::Whitespace(Whitespace::Space),
Token::Number(String::from("1e-10"), false),
Token::Comma,
Token::Whitespace(Whitespace::Space),
Token::Number(String::from("1e+10"), false),
Token::Comma,
Token::Whitespace(Whitespace::Space),
Token::Number(String::from("1"), false),
Token::make_word("ea", None),
Token::Comma,
Token::Whitespace(Whitespace::Space),
Token::Number(String::from("1e-10"), false),
Token::make_word("a", None),
Token::Comma,
Token::Whitespace(Whitespace::Space),
Token::Number(String::from("1e-10"), false),
Token::Minus,
Token::Number(String::from("10"), false),
];
compare(expected, tokens);
}
#[test]
fn tokenize_scalar_function() {
let sql = String::from("SELECT sqrt(1)");