mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-04 06:18:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Optionally parse numbers into BigDecimals

Browse source 
With `--features bigdecimal`, parse numbers into BigDecimals instead of
leaving them as strings.
This commit is contained in:
Nikhil Benesch 2019-08-13 14:42:31 -04:00
parent b5621c0fe8
commit a0aca824e8
No known key found for this signature in database
GPG key ID: FCF98542083C5A69
7 changed files with 95 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.travis.yml
View file

@ -49,6 +49,7 @@ script:
- travis-cargo clippy -- --all-targets --all-features -- -D warnings - travis-cargo clippy -- --all-targets --all-features -- -D warnings
- travis-cargo build - travis-cargo build
- travis-cargo test - travis-cargo test
- travis-cargo test -- all-features
- cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"') - cargo +nightly fmt -- --check --config-path <(echo 'license_template_path = "HEADER"')
after_success: after_success:

1
Cargo.toml
View file

@ -19,6 +19,7 @@ name = "sqlparser"
path = "src/lib.rs" path = "src/lib.rs"
[dependencies] [dependencies]
bigdecimal = { version = "0.1.0", optional = true }
log = "0.4.5" log = "0.4.5"
[dev-dependencies] [dev-dependencies]

5
src/ast/value.rs
View file

@ -10,13 +10,18 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#[cfg(feature = "bigdecimal")]
use bigdecimal::BigDecimal;
use std::fmt; use std::fmt;
/// Primitive SQL values such as number and string /// Primitive SQL values such as number and string
#[derive(Debug, Clone, PartialEq, Eq, Hash)] #[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum Value { pub enum Value {
/// Numeric literal /// Numeric literal
#[cfg(not(feature = "bigdecimal"))]
Number(String), Number(String),
#[cfg(feature = "bigdecimal")]
Number(BigDecimal),
/// 'string value' /// 'string value'
SingleQuotedString(String), SingleQuotedString(String),
/// N'string value' /// N'string value'

25
src/parser.rs
View file

@ -1183,7 +1183,13 @@ impl Parser {
return parser_err!(format!("No value parser for keyword {}", k.keyword)); return parser_err!(format!("No value parser for keyword {}", k.keyword));
} }
}, },
Token::Number(ref n) => Ok(Value::Number(n.to_string())), // The call to n.parse() returns a bigdecimal when the
// bigdecimal feature is enabled, and is otherwise a no-op
// (i.e., it returns the input string).
Token::Number(ref n) => match n.parse() {
Ok(n) => Ok(Value::Number(n)),
Err(e) => parser_err!(format!("Could not parse '{}' as number: {}", n, e)),
},
Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())), Token::SingleQuotedString(ref s) => Ok(Value::SingleQuotedString(s.to_string())),
Token::NationalStringLiteral(ref s) => { Token::NationalStringLiteral(ref s) => {
Ok(Value::NationalStringLiteral(s.to_string())) Ok(Value::NationalStringLiteral(s.to_string()))
@ -1195,6 +1201,16 @@ impl Parser {
} }
} }
pub fn parse_number_value(&mut self) -> Result<Value, ParserError> {
match self.parse_value()? {
v @ Value::Number(_) => Ok(v),
_ => {
self.prev_token();
self.expected("literal number", self.peek_token())
}
}
}
/// Parse an unsigned literal integer/long /// Parse an unsigned literal integer/long
pub fn parse_literal_uint(&mut self) -> Result<u64, ParserError> { pub fn parse_literal_uint(&mut self) -> Result<u64, ParserError> {
match self.next_token() { match self.next_token() {
@ -1856,16 +1872,13 @@ impl Parser {
if self.parse_keyword("ALL") { if self.parse_keyword("ALL") {
Ok(None) Ok(None)
} else { } else {
self.parse_literal_uint() Ok(Some(Expr::Value(self.parse_number_value()?)))
.map(|n| Some(Expr::Value(Value::Number(n.to_string()))))
} }
} }
/// Parse an OFFSET clause /// Parse an OFFSET clause
pub fn parse_offset(&mut self) -> Result<Expr, ParserError> { pub fn parse_offset(&mut self) -> Result<Expr, ParserError> {
let value = self let value = Expr::Value(self.parse_number_value()?);
.parse_literal_uint()
.map(|n| Expr::Value(Value::Number(n.to_string())))?;
self.expect_one_of_keywords(&["ROW", "ROWS"])?; self.expect_one_of_keywords(&["ROW", "ROWS"])?;
Ok(value) Ok(value)
} }

4
src/test_utils.rs
View file

@ -136,3 +136,7 @@ pub fn expr_from_projection(item: &SelectItem) -> &Expr {
_ => panic!("Expected UnnamedExpr"), _ => panic!("Expected UnnamedExpr"),
} }
} }
pub fn number(n: &'static str) -> Value {
Value::Number(n.parse().unwrap())
}

112
tests/sqlparser_common.rs
View file

@ -22,14 +22,14 @@ use matches::assert_matches;
use sqlparser::ast::*; use sqlparser::ast::*;
use sqlparser::parser::*; use sqlparser::parser::*;
use sqlparser::test_utils::{all_dialects, expr_from_projection, only}; use sqlparser::test_utils::{all_dialects, expr_from_projection, number, only};
#[test] #[test]
fn parse_insert_values() { fn parse_insert_values() {
let row = vec![ let row = vec![
Expr::Value(Value::Number("1".into())), Expr::Value(number("1")),
Expr::Value(Value::Number("2".into())), Expr::Value(number("2")),
Expr::Value(Value::Number("3".into())), Expr::Value(number("3")),
]; ];
let rows1 = vec![row.clone()]; let rows1 = vec![row.clone()];
let rows2 = vec![row.clone(), row]; let rows2 = vec![row.clone(), row];
@ -107,15 +107,15 @@ fn parse_update() {
vec![ vec![
Assignment { Assignment {
id: "a".into(), id: "a".into(),
value: Expr::Value(Value::Number("1".into())), value: Expr::Value(number("1")),
}, },
Assignment { Assignment {
id: "b".into(), id: "b".into(),
value: Expr::Value(Value::Number("2".into())), value: Expr::Value(number("2")),
}, },
Assignment { Assignment {
id: "c".into(), id: "c".into(),
value: Expr::Value(Value::Number("3".into())), value: Expr::Value(number("3")),
}, },
] ]
); );
@ -181,7 +181,7 @@ fn parse_where_delete_statement() {
Expr::BinaryOp { Expr::BinaryOp {
left: Box::new(Expr::Identifier("name".to_string())), left: Box::new(Expr::Identifier("name".to_string())),
op: Eq, op: Eq,
right: Box::new(Expr::Value(Value::Number("5".into()))), right: Box::new(Expr::Value(number("5"))),
}, },
selection.unwrap(), selection.unwrap(),
); );
@ -205,17 +205,17 @@ fn parse_simple_select() {
assert_eq!(false, select.distinct); assert_eq!(false, select.distinct);
assert_eq!(3, select.projection.len()); assert_eq!(3, select.projection.len());
let select = verified_query(sql); let select = verified_query(sql);
assert_eq!(Some(Expr::Value(Value::Number("5".into()))), select.limit); assert_eq!(Some(Expr::Value(number("5"))), select.limit);
} }
#[test] #[test]
fn parse_limit_is_not_an_alias() { fn parse_limit_is_not_an_alias() {
// In dialects supporting LIMIT it shouldn't be parsed as a table alias // In dialects supporting LIMIT it shouldn't be parsed as a table alias
let ast = verified_query("SELECT id FROM customer LIMIT 1"); let ast = verified_query("SELECT id FROM customer LIMIT 1");
assert_eq!(Some(Expr::Value(Value::Number("1".into()))), ast.limit); assert_eq!(Some(Expr::Value(number("1"))), ast.limit);
let ast = verified_query("SELECT 1 LIMIT 5"); let ast = verified_query("SELECT 1 LIMIT 5");
assert_eq!(Some(Expr::Value(Value::Number("5".into()))), ast.limit); assert_eq!(Some(Expr::Value(number("5"))), ast.limit);
} }
#[test] #[test]
@ -286,7 +286,7 @@ fn parse_column_aliases() {
} = only(&select.projection) } = only(&select.projection)
{ {
assert_eq!(&BinaryOperator::Plus, op); assert_eq!(&BinaryOperator::Plus, op);
assert_eq!(&Expr::Value(Value::Number("1".into())), right.as_ref()); assert_eq!(&Expr::Value(number("1")), right.as_ref());
assert_eq!("newname", alias); assert_eq!("newname", alias);
} else { } else {
panic!("Expected ExprWithAlias") panic!("Expected ExprWithAlias")
@ -426,6 +426,20 @@ fn parse_escaped_single_quote_string_predicate() {
); );
} }
#[test]
fn parse_number() {
let expr = verified_expr("1.0");
#[cfg(feature = "bigdecimal")]
assert_eq!(
expr,
Expr::Value(Value::Number(bigdecimal::BigDecimal::from(1)))
);
#[cfg(not(feature = "bigdecimal"))]
assert_eq!(expr, Expr::Value(Value::Number("1.0".into())));
}
#[test] #[test]
fn parse_compound_expr_1() { fn parse_compound_expr_1() {
use self::BinaryOperator::*; use self::BinaryOperator::*;
@ -527,9 +541,9 @@ fn parse_not_precedence() {
Expr::UnaryOp { Expr::UnaryOp {
op: UnaryOperator::Not, op: UnaryOperator::Not,
expr: Box::new(Expr::Between { expr: Box::new(Expr::Between {
expr: Box::new(Expr::Value(Value::Number("1".into()))), expr: Box::new(Expr::Value(number("1"))),
low: Box::new(Expr::Value(Value::Number("1".into()))), low: Box::new(Expr::Value(number("1"))),
high: Box::new(Expr::Value(Value::Number("2".into()))), high: Box::new(Expr::Value(number("2"))),
negated: true, negated: true,
}), }),
}, },
@ -658,8 +672,8 @@ fn parse_between() {
assert_eq!( assert_eq!(
Expr::Between { Expr::Between {
expr: Box::new(Expr::Identifier("age".to_string())), expr: Box::new(Expr::Identifier("age".to_string())),
low: Box::new(Expr::Value(Value::Number("25".into()))), low: Box::new(Expr::Value(number("25"))),
high: Box::new(Expr::Value(Value::Number("32".into()))), high: Box::new(Expr::Value(number("32"))),
negated, negated,
}, },
select.selection.unwrap() select.selection.unwrap()
@ -676,16 +690,16 @@ fn parse_between_with_expr() {
let select = verified_only_select(sql); let select = verified_only_select(sql);
assert_eq!( assert_eq!(
Expr::IsNull(Box::new(Expr::Between { Expr::IsNull(Box::new(Expr::Between {
expr: Box::new(Expr::Value(Value::Number("1".into()))), expr: Box::new(Expr::Value(number("1"))),
low: Box::new(Expr::BinaryOp { low: Box::new(Expr::BinaryOp {
left: Box::new(Expr::Value(Value::Number("1".into()))), left: Box::new(Expr::Value(number("1"))),
op: Plus, op: Plus,
right: Box::new(Expr::Value(Value::Number("2".into()))), right: Box::new(Expr::Value(number("2"))),
}), }),
high: Box::new(Expr::BinaryOp { high: Box::new(Expr::BinaryOp {
left: Box::new(Expr::Value(Value::Number("3".into()))), left: Box::new(Expr::Value(number("3"))),
op: Plus, op: Plus,
right: Box::new(Expr::Value(Value::Number("4".into()))), right: Box::new(Expr::Value(number("4"))),
}), }),
negated: false, negated: false,
})), })),
@ -697,19 +711,19 @@ fn parse_between_with_expr() {
assert_eq!( assert_eq!(
Expr::BinaryOp { Expr::BinaryOp {
left: Box::new(Expr::BinaryOp { left: Box::new(Expr::BinaryOp {
left: Box::new(Expr::Value(Value::Number("1".into()))), left: Box::new(Expr::Value(number("1"))),
op: BinaryOperator::Eq, op: BinaryOperator::Eq,
right: Box::new(Expr::Value(Value::Number("1".into()))), right: Box::new(Expr::Value(number("1"))),
}), }),
op: BinaryOperator::And, op: BinaryOperator::And,
right: Box::new(Expr::Between { right: Box::new(Expr::Between {
expr: Box::new(Expr::BinaryOp { expr: Box::new(Expr::BinaryOp {
left: Box::new(Expr::Value(Value::Number("1".into()))), left: Box::new(Expr::Value(number("1"))),
op: BinaryOperator::Plus, op: BinaryOperator::Plus,
right: Box::new(Expr::Identifier("x".to_string())), right: Box::new(Expr::Identifier("x".to_string())),
}), }),
low: Box::new(Expr::Value(Value::Number("1".into()))), low: Box::new(Expr::Value(number("1"))),
high: Box::new(Expr::Value(Value::Number("2".into()))), high: Box::new(Expr::Value(number("2"))),
negated: false, negated: false,
}), }),
}, },
@ -763,7 +777,7 @@ fn parse_select_order_by_limit() {
], ],
select.order_by select.order_by
); );
assert_eq!(Some(Expr::Value(Value::Number("2".into()))), select.limit); assert_eq!(Some(Expr::Value(number("2"))), select.limit);
} }
#[test] #[test]
@ -792,7 +806,7 @@ fn parse_select_having() {
distinct: false distinct: false
})), })),
op: BinaryOperator::Gt, op: BinaryOperator::Gt,
right: Box::new(Expr::Value(Value::Number("1".into()))) right: Box::new(Expr::Value(number("1")))
}), }),
select.having select.having
); );
@ -988,7 +1002,7 @@ fn parse_create_table_with_options() {
}, },
SqlOption { SqlOption {
name: "a".into(), name: "a".into(),
value: Value::Number("123".into()) value: number("123")
}, },
], ],
with_options with_options
@ -1186,11 +1200,11 @@ fn parse_literal_decimal() {
let select = verified_only_select(sql); let select = verified_only_select(sql);
assert_eq!(2, select.projection.len()); assert_eq!(2, select.projection.len());
assert_eq!( assert_eq!(
&Expr::Value(Value::Number("0.300000000000000004".into())), &Expr::Value(number("0.300000000000000004")),
expr_from_projection(&select.projection[0]), expr_from_projection(&select.projection[0]),
); );
assert_eq!( assert_eq!(
&Expr::Value(Value::Number("9007199254740993.0".into())), &Expr::Value(number("9007199254740993.0")),
expr_from_projection(&select.projection[1]), expr_from_projection(&select.projection[1]),
) )
} }
@ -1438,12 +1452,12 @@ fn parse_searched_case_expr() {
BinaryOp { BinaryOp {
left: Box::new(Identifier("bar".to_string())), left: Box::new(Identifier("bar".to_string())),
op: Eq, op: Eq,
right: Box::new(Expr::Value(Value::Number("0".into()))) right: Box::new(Expr::Value(number("0")))
}, },
BinaryOp { BinaryOp {
left: Box::new(Identifier("bar".to_string())), left: Box::new(Identifier("bar".to_string())),
op: GtEq, op: GtEq,
right: Box::new(Expr::Value(Value::Number("0".into()))) right: Box::new(Expr::Value(number("0")))
} }
], ],
results: vec![ results: vec![
@ -1468,7 +1482,7 @@ fn parse_simple_case_expr() {
assert_eq!( assert_eq!(
&Case { &Case {
operand: Some(Box::new(Identifier("foo".to_string()))), operand: Some(Box::new(Identifier("foo".to_string()))),
conditions: vec![Expr::Value(Value::Number("1".into()))], conditions: vec![Expr::Value(number("1"))],
results: vec![Expr::Value(Value::SingleQuotedString("Y".to_string())),], results: vec![Expr::Value(Value::SingleQuotedString("Y".to_string())),],
else_result: Some(Box::new(Expr::Value(Value::SingleQuotedString( else_result: Some(Box::new(Expr::Value(Value::SingleQuotedString(
"N".to_string() "N".to_string()
@ -2080,7 +2094,7 @@ fn parse_create_view_with_options() {
}, },
SqlOption { SqlOption {
name: "a".into(), name: "a".into(),
value: Value::Number("123".into()) value: number("123")
}, },
], ],
with_options with_options
@ -2214,26 +2228,26 @@ fn parse_invalid_subquery_without_parens() {
#[test] #[test]
fn parse_offset() { fn parse_offset() {
let ast = verified_query("SELECT foo FROM bar OFFSET 2 ROWS"); let ast = verified_query("SELECT foo FROM bar OFFSET 2 ROWS");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
let ast = verified_query("SELECT foo FROM bar WHERE foo = 4 OFFSET 2 ROWS"); let ast = verified_query("SELECT foo FROM bar WHERE foo = 4 OFFSET 2 ROWS");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
let ast = verified_query("SELECT foo FROM bar ORDER BY baz OFFSET 2 ROWS"); let ast = verified_query("SELECT foo FROM bar ORDER BY baz OFFSET 2 ROWS");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
let ast = verified_query("SELECT foo FROM bar WHERE foo = 4 ORDER BY baz OFFSET 2 ROWS"); let ast = verified_query("SELECT foo FROM bar WHERE foo = 4 ORDER BY baz OFFSET 2 ROWS");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
let ast = verified_query("SELECT foo FROM (SELECT * FROM bar OFFSET 2 ROWS) OFFSET 2 ROWS"); let ast = verified_query("SELECT foo FROM (SELECT * FROM bar OFFSET 2 ROWS) OFFSET 2 ROWS");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
match ast.body { match ast.body {
SetExpr::Select(s) => match only(s.from).relation { SetExpr::Select(s) => match only(s.from).relation {
TableFactor::Derived { subquery, .. } => { TableFactor::Derived { subquery, .. } => {
assert_eq!(subquery.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(subquery.offset, Some(Expr::Value(number("2"))));
} }
_ => panic!("Test broke"), _ => panic!("Test broke"),
}, },
_ => panic!("Test broke"), _ => panic!("Test broke"),
} }
let ast = verified_query("SELECT 'foo' OFFSET 0 ROWS"); let ast = verified_query("SELECT 'foo' OFFSET 0 ROWS");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("0".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("0"))));
} }
#[test] #[test]
@ -2249,7 +2263,7 @@ fn parse_fetch() {
let fetch_first_two_rows_only = Some(Fetch { let fetch_first_two_rows_only = Some(Fetch {
with_ties: false, with_ties: false,
percent: false, percent: false,
quantity: Some(Expr::Value(Value::Number("2".into()))), quantity: Some(Expr::Value(number("2"))),
}); });
let ast = verified_query("SELECT foo FROM bar FETCH FIRST 2 ROWS ONLY"); let ast = verified_query("SELECT foo FROM bar FETCH FIRST 2 ROWS ONLY");
assert_eq!(ast.fetch, fetch_first_two_rows_only); assert_eq!(ast.fetch, fetch_first_two_rows_only);
@ -2276,7 +2290,7 @@ fn parse_fetch() {
Some(Fetch { Some(Fetch {
with_ties: true, with_ties: true,
percent: false, percent: false,
quantity: Some(Expr::Value(Value::Number("2".into()))), quantity: Some(Expr::Value(number("2"))),
}) })
); );
let ast = verified_query("SELECT foo FROM bar FETCH FIRST 50 PERCENT ROWS ONLY"); let ast = verified_query("SELECT foo FROM bar FETCH FIRST 50 PERCENT ROWS ONLY");
@ -2285,13 +2299,13 @@ fn parse_fetch() {
Some(Fetch { Some(Fetch {
with_ties: false, with_ties: false,
percent: true, percent: true,
quantity: Some(Expr::Value(Value::Number("50".into()))), quantity: Some(Expr::Value(number("50"))),
}) })
); );
let ast = verified_query( let ast = verified_query(
"SELECT foo FROM bar WHERE foo = 4 ORDER BY baz OFFSET 2 ROWS FETCH FIRST 2 ROWS ONLY", "SELECT foo FROM bar WHERE foo = 4 ORDER BY baz OFFSET 2 ROWS FETCH FIRST 2 ROWS ONLY",
); );
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
assert_eq!(ast.fetch, fetch_first_two_rows_only); assert_eq!(ast.fetch, fetch_first_two_rows_only);
let ast = verified_query( let ast = verified_query(
"SELECT foo FROM (SELECT * FROM bar FETCH FIRST 2 ROWS ONLY) FETCH FIRST 2 ROWS ONLY", "SELECT foo FROM (SELECT * FROM bar FETCH FIRST 2 ROWS ONLY) FETCH FIRST 2 ROWS ONLY",
@ -2307,12 +2321,12 @@ fn parse_fetch() {
_ => panic!("Test broke"), _ => panic!("Test broke"),
} }
let ast = verified_query("SELECT foo FROM (SELECT * FROM bar OFFSET 2 ROWS FETCH FIRST 2 ROWS ONLY) OFFSET 2 ROWS FETCH FIRST 2 ROWS ONLY"); let ast = verified_query("SELECT foo FROM (SELECT * FROM bar OFFSET 2 ROWS FETCH FIRST 2 ROWS ONLY) OFFSET 2 ROWS FETCH FIRST 2 ROWS ONLY");
assert_eq!(ast.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(ast.offset, Some(Expr::Value(number("2"))));
assert_eq!(ast.fetch, fetch_first_two_rows_only); assert_eq!(ast.fetch, fetch_first_two_rows_only);
match ast.body { match ast.body {
SetExpr::Select(s) => match only(s.from).relation { SetExpr::Select(s) => match only(s.from).relation {
TableFactor::Derived { subquery, .. } => { TableFactor::Derived { subquery, .. } => {
assert_eq!(subquery.offset, Some(Expr::Value(Value::Number("2".into())))); assert_eq!(subquery.offset, Some(Expr::Value(number("2"))));
assert_eq!(subquery.fetch, fetch_first_two_rows_only); assert_eq!(subquery.fetch, fetch_first_two_rows_only);
} }
_ => panic!("Test broke"), _ => panic!("Test broke"),

4
tests/sqlparser_postgres.rs
View file

@ -163,7 +163,7 @@ fn parse_create_table_with_defaults() {
vec![ vec![
SqlOption { SqlOption {
name: "fillfactor".into(), name: "fillfactor".into(),
value: Value::Number("20".into()) value: number("20")
}, },
SqlOption { SqlOption {
name: "user_catalog_table".into(), name: "user_catalog_table".into(),
@ -171,7 +171,7 @@ fn parse_create_table_with_defaults() {
}, },
SqlOption { SqlOption {
name: "autovacuum_vacuum_threshold".into(), name: "autovacuum_vacuum_threshold".into(),
value: Value::Number("100".into()) value: number("100")
}, },
] ]
); );