mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-07-07 17:04:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Fix the parsing result for the special double number (#1621)

Browse source
This commit is contained in:
Jax Liu 2024-12-28 21:16:30 +08:00 • committed by GitHub
parent 6daa4b059c
commit d0d4153137
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 410 additions and 265 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/dialect/mysql.rs
View file

@ -113,7 +113,7 @@ fn parse_lock_tables(parser: &mut Parser) -> Result<Statement, ParserError> {
// tbl_name [[AS] alias] lock_type
fn parse_lock_table(parser: &mut Parser) -> Result<LockTable, ParserError> {
let table = parser.parse_identifier(false)?;
let table = parser.parse_identifier()?;
let alias =
parser.parse_optional_alias(&[Keyword::READ, Keyword::WRITE, Keyword::LOW_PRIORITY])?;
let lock_type = parse_lock_tables_type(parser)?;

2
src/dialect/postgresql.rs
View file

@ -268,7 +268,7 @@ pub fn parse_create_type_as_enum(
return parser.expected("'(' after CREATE TYPE AS ENUM", parser.peek_token());
}
let labels = parser.parse_comma_separated0(|p| p.parse_identifier(false), Token::RParen)?;
let labels = parser.parse_comma_separated0(|p| p.parse_identifier(), Token::RParen)?;
parser.expect_token(&Token::RParen)?;
Ok(Statement::CreateType {

8
src/dialect/snowflake.rs
View file

@ -300,7 +300,7 @@ pub fn parse_create_table(
parser.expect_keyword_is(Keyword::BY)?;
parser.expect_token(&Token::LParen)?;
let cluster_by = Some(WrappedCollection::Parentheses(
parser.parse_comma_separated(|p| p.parse_identifier(false))?,
parser.parse_comma_separated(|p| p.parse_identifier())?,
));
parser.expect_token(&Token::RParen)?;
@ -369,7 +369,7 @@ pub fn parse_create_table(
let policy = parser.parse_object_name(false)?;
parser.expect_keyword_is(Keyword::ON)?;
parser.expect_token(&Token::LParen)?;
let columns = parser.parse_comma_separated(|p| p.parse_identifier(false))?;
let columns = parser.parse_comma_separated(|p| p.parse_identifier())?;
parser.expect_token(&Token::RParen)?;
builder =
@ -887,10 +887,10 @@ fn parse_column_policy_property(
parser: &mut Parser,
with: bool,
) -> Result<ColumnPolicyProperty, ParserError> {
let policy_name = parser.parse_identifier(false)?;
let policy_name = parser.parse_identifier()?;
let using_columns = if parser.parse_keyword(Keyword::USING) {
parser.expect_token(&Token::LParen)?;
let columns = parser.parse_comma_separated(|p| p.parse_identifier(false))?;
let columns = parser.parse_comma_separated(|p| p.parse_identifier())?;
parser.expect_token(&Token::RParen)?;
Some(columns)
} else {

16
src/parser/alter.rs
View file

@ -51,13 +51,13 @@ impl Parser<'_> {
///
/// [PostgreSQL](https://www.postgresql.org/docs/current/sql-alterpolicy.html)
pub fn parse_alter_policy(&mut self) -> Result<Statement, ParserError> {
let name = self.parse_identifier(false)?;
let name = self.parse_identifier()?;
self.expect_keyword_is(Keyword::ON)?;
let table_name = self.parse_object_name(false)?;
if self.parse_keyword(Keyword::RENAME) {
self.expect_keyword_is(Keyword::TO)?;
let new_name = self.parse_identifier(false)?;
let new_name = self.parse_identifier()?;
Ok(Statement::AlterPolicy {
name,
table_name,
@ -100,17 +100,17 @@ impl Parser<'_> {
}
fn parse_mssql_alter_role(&mut self) -> Result<Statement, ParserError> {
let role_name = self.parse_identifier(false)?;
let role_name = self.parse_identifier()?;
let operation = if self.parse_keywords(&[Keyword::ADD, Keyword::MEMBER]) {
let member_name = self.parse_identifier(false)?;
let member_name = self.parse_identifier()?;
AlterRoleOperation::AddMember { member_name }
} else if self.parse_keywords(&[Keyword::DROP, Keyword::MEMBER]) {
let member_name = self.parse_identifier(false)?;
let member_name = self.parse_identifier()?;
AlterRoleOperation::DropMember { member_name }
} else if self.parse_keywords(&[Keyword::WITH, Keyword::NAME]) {
if self.consume_token(&Token::Eq) {
let role_name = self.parse_identifier(false)?;
let role_name = self.parse_identifier()?;
AlterRoleOperation::RenameRole { role_name }
} else {
return self.expected("= after WITH NAME ", self.peek_token());
@ -126,7 +126,7 @@ impl Parser<'_> {
}
fn parse_pg_alter_role(&mut self) -> Result<Statement, ParserError> {
let role_name = self.parse_identifier(false)?;
let role_name = self.parse_identifier()?;
// [ IN DATABASE _`database_name`_ ]
let in_database = if self.parse_keywords(&[Keyword::IN, Keyword::DATABASE]) {
@ -137,7 +137,7 @@ impl Parser<'_> {
let operation = if self.parse_keyword(Keyword::RENAME) {
if self.parse_keyword(Keyword::TO) {
let role_name = self.parse_identifier(false)?;
let role_name = self.parse_identifier()?;
AlterRoleOperation::RenameRole { role_name }
} else {
return self.expected("TO after RENAME", self.peek_token());

460
src/parser/mod.rs
View file

File diff suppressed because it is too large Load diff

45
src/tokenizer.rs
View file

@ -1144,30 +1144,16 @@ impl<'a> Tokenizer<'a> {
// match one period
if let Some('.') = chars.peek() {
// Check if this actually is a float point number
let mut char_clone = chars.peekable.clone();
char_clone.next();
// Next char should be a digit, otherwise, it is not a float point number
if char_clone
.peek()
.map(|c| c.is_ascii_digit())
.unwrap_or(false)
{
s.push('.');
chars.next();
} else if !s.is_empty() {
// Number might be part of period separated construct. Keep the period for next token
// e.g. a-12.b
return Ok(Some(Token::Number(s, false)));
} else {
// No number -> Token::Period
chars.next();
return Ok(Some(Token::Period));
}
s.push('.');
chars.next();
}
s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
// No number -> Token::Period
if s == "." {
return Ok(Some(Token::Period));
}
let mut exponent_part = String::new();
// Parse exponent as number
if chars.peek() == Some(&'e') || chars.peek() == Some(&'E') {
@ -2199,23 +2185,6 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_select_float_hyphenated_identifier() {
let sql = String::from("SELECT a-12.b");
let dialect = GenericDialect {};
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::make_word("a", None),
Token::Minus,
Token::Number(String::from("12"), false),
Token::Period,
Token::make_word("b", None),
];
compare(expected, tokens);
}
#[test]
fn tokenize_clickhouse_double_equal() {
let sql = String::from("SELECT foo=='1'");

142
tests/sqlparser_common.rs
View file

@ -2964,6 +2964,113 @@ fn test_compound_expr() {
}
}
#[test]
fn test_double_value() {
let dialects = all_dialects();
let test_cases = vec![
gen_number_case_with_sign("0."),
gen_number_case_with_sign("0.0"),
gen_number_case_with_sign("0000."),
gen_number_case_with_sign("0000.00"),
gen_number_case_with_sign(".0"),
gen_number_case_with_sign(".00"),
gen_number_case_with_sign("0e0"),
gen_number_case_with_sign("0e+0"),
gen_number_case_with_sign("0e-0"),
gen_number_case_with_sign("0.e-0"),
gen_number_case_with_sign("0.e+0"),
gen_number_case_with_sign(".0e-0"),
gen_number_case_with_sign(".0e+0"),
gen_number_case_with_sign("00.0e+0"),
gen_number_case_with_sign("00.0e-0"),
];
for (input, expected) in test_cases {
for (i, expr) in input.iter().enumerate() {
if let Statement::Query(query) =
dialects.one_statement_parses_to(&format!("SELECT {}", expr), "")
{
if let SetExpr::Select(select) = *query.body {
assert_eq!(expected[i], select.projection[0]);
} else {
panic!("Expected a SELECT statement");
}
} else {
panic!("Expected a SELECT statement");
}
}
}
}
fn gen_number_case(value: &str) -> (Vec<String>, Vec<SelectItem>) {
let input = vec![
value.to_string(),
format!("{} col_alias", value),
format!("{} AS col_alias", value),
];
let expected = vec![
SelectItem::UnnamedExpr(Expr::Value(number(value))),
SelectItem::ExprWithAlias {
expr: Expr::Value(number(value)),
alias: Ident::new("col_alias"),
},
SelectItem::ExprWithAlias {
expr: Expr::Value(number(value)),
alias: Ident::new("col_alias"),
},
];
(input, expected)
}
fn gen_sign_number_case(value: &str, op: UnaryOperator) -> (Vec<String>, Vec<SelectItem>) {
match op {
UnaryOperator::Plus | UnaryOperator::Minus => {}
_ => panic!("Invalid sign"),
}
let input = vec![
format!("{}{}", op, value),
format!("{}{} col_alias", op, value),
format!("{}{} AS col_alias", op, value),
];
let expected = vec![
SelectItem::UnnamedExpr(Expr::UnaryOp {
op,
expr: Box::new(Expr::Value(number(value))),
}),
SelectItem::ExprWithAlias {
expr: Expr::UnaryOp {
op,
expr: Box::new(Expr::Value(number(value))),
},
alias: Ident::new("col_alias"),
},
SelectItem::ExprWithAlias {
expr: Expr::UnaryOp {
op,
expr: Box::new(Expr::Value(number(value))),
},
alias: Ident::new("col_alias"),
},
];
(input, expected)
}
/// generate the test cases for signed and unsigned numbers
/// For example, given "0.0", the test cases will be:
/// - "0.0"
/// - "+0.0"
/// - "-0.0"
fn gen_number_case_with_sign(number: &str) -> (Vec<String>, Vec<SelectItem>) {
let (mut input, mut expected) = gen_number_case(number);
for op in [UnaryOperator::Plus, UnaryOperator::Minus] {
let (input_sign, expected_sign) = gen_sign_number_case(number, op);
input.extend(input_sign);
expected.extend(expected_sign);
}
(input, expected)
}
#[test]
fn parse_negative_value() {
let sql1 = "SELECT -1";
@ -12470,6 +12577,41 @@ fn parse_composite_access_expr() {
all_dialects_where(|d| d.supports_struct_literal()).verified_stmt(
"SELECT * FROM t WHERE STRUCT(STRUCT(1 AS a, NULL AS b) AS c, NULL AS d).c.a IS NOT NULL",
);
let support_struct = all_dialects_where(|d| d.supports_struct_literal());
let stmt = support_struct
.verified_only_select("SELECT STRUCT(STRUCT(1 AS a, NULL AS b) AS c, NULL AS d).c.a");
let expected = SelectItem::UnnamedExpr(Expr::CompoundFieldAccess {
root: Box::new(Expr::Struct {
values: vec![
Expr::Named {
name: Ident::new("c"),
expr: Box::new(Expr::Struct {
values: vec![
Expr::Named {
name: Ident::new("a"),
expr: Box::new(Expr::Value(Number("1".parse().unwrap(), false))),
},
Expr::Named {
name: Ident::new("b"),
expr: Box::new(Expr::Value(Value::Null)),
},
],
fields: vec![],
}),
},
Expr::Named {
name: Ident::new("d"),
expr: Box::new(Expr::Value(Value::Null)),
},
],
fields: vec![],
}),
access_chain: vec![
AccessExpr::Dot(Expr::Identifier(Ident::new("c"))),
AccessExpr::Dot(Expr::Identifier(Ident::new("a"))),
],
});
assert_eq!(stmt.projection[0], expected);
}
#[test]