Fix the parsing result for the special double number (#1621)

src/dialect/mysql.rs

@@ -113,7 +113,7 @@ fn parse_lock_tables(parser: &mut Parser) -> Result<Statement, ParserError> {
 
 // tbl_name [[AS] alias] lock_type
 fn parse_lock_table(parser: &mut Parser) -> Result<LockTable, ParserError> {
-    let table = parser.parse_identifier(false)?;
+    let table = parser.parse_identifier()?;
     let alias =
         parser.parse_optional_alias(&[Keyword::READ, Keyword::WRITE, Keyword::LOW_PRIORITY])?;
     let lock_type = parse_lock_tables_type(parser)?;

src/dialect/postgresql.rs

@@ -268,7 +268,7 @@ pub fn parse_create_type_as_enum(
         return parser.expected("'(' after CREATE TYPE AS ENUM", parser.peek_token());
     }
 
-    let labels = parser.parse_comma_separated0(|p| p.parse_identifier(false), Token::RParen)?;
+    let labels = parser.parse_comma_separated0(|p| p.parse_identifier(), Token::RParen)?;
     parser.expect_token(&Token::RParen)?;
 
     Ok(Statement::CreateType {

src/dialect/snowflake.rs

@@ -300,7 +300,7 @@ pub fn parse_create_table(
                     parser.expect_keyword_is(Keyword::BY)?;
                     parser.expect_token(&Token::LParen)?;
                     let cluster_by = Some(WrappedCollection::Parentheses(
-                        parser.parse_comma_separated(|p| p.parse_identifier(false))?,
+                        parser.parse_comma_separated(|p| p.parse_identifier())?,
                     ));
                     parser.expect_token(&Token::RParen)?;
 
@@ -369,7 +369,7 @@ pub fn parse_create_table(
                     let policy = parser.parse_object_name(false)?;
                     parser.expect_keyword_is(Keyword::ON)?;
                     parser.expect_token(&Token::LParen)?;
-                    let columns = parser.parse_comma_separated(|p| p.parse_identifier(false))?;
+                    let columns = parser.parse_comma_separated(|p| p.parse_identifier())?;
                     parser.expect_token(&Token::RParen)?;
 
                     builder =
@@ -887,10 +887,10 @@ fn parse_column_policy_property(
     parser: &mut Parser,
     with: bool,
 ) -> Result<ColumnPolicyProperty, ParserError> {
-    let policy_name = parser.parse_identifier(false)?;
+    let policy_name = parser.parse_identifier()?;
     let using_columns = if parser.parse_keyword(Keyword::USING) {
         parser.expect_token(&Token::LParen)?;
-        let columns = parser.parse_comma_separated(|p| p.parse_identifier(false))?;
+        let columns = parser.parse_comma_separated(|p| p.parse_identifier())?;
         parser.expect_token(&Token::RParen)?;
         Some(columns)
     } else {

src/parser/alter.rs

@@ -51,13 +51,13 @@ impl Parser<'_> {
     ///
     /// [PostgreSQL](https://www.postgresql.org/docs/current/sql-alterpolicy.html)
     pub fn parse_alter_policy(&mut self) -> Result<Statement, ParserError> {
-        let name = self.parse_identifier(false)?;
+        let name = self.parse_identifier()?;
         self.expect_keyword_is(Keyword::ON)?;
         let table_name = self.parse_object_name(false)?;
 
         if self.parse_keyword(Keyword::RENAME) {
             self.expect_keyword_is(Keyword::TO)?;
-            let new_name = self.parse_identifier(false)?;
+            let new_name = self.parse_identifier()?;
             Ok(Statement::AlterPolicy {
                 name,
                 table_name,
@@ -100,17 +100,17 @@ impl Parser<'_> {
     }
 
     fn parse_mssql_alter_role(&mut self) -> Result<Statement, ParserError> {
-        let role_name = self.parse_identifier(false)?;
+        let role_name = self.parse_identifier()?;
 
         let operation = if self.parse_keywords(&[Keyword::ADD, Keyword::MEMBER]) {
-            let member_name = self.parse_identifier(false)?;
+            let member_name = self.parse_identifier()?;
             AlterRoleOperation::AddMember { member_name }
         } else if self.parse_keywords(&[Keyword::DROP, Keyword::MEMBER]) {
-            let member_name = self.parse_identifier(false)?;
+            let member_name = self.parse_identifier()?;
             AlterRoleOperation::DropMember { member_name }
         } else if self.parse_keywords(&[Keyword::WITH, Keyword::NAME]) {
             if self.consume_token(&Token::Eq) {
-                let role_name = self.parse_identifier(false)?;
+                let role_name = self.parse_identifier()?;
                 AlterRoleOperation::RenameRole { role_name }
             } else {
                 return self.expected("= after WITH NAME ", self.peek_token());
@@ -126,7 +126,7 @@ impl Parser<'_> {
     }
 
     fn parse_pg_alter_role(&mut self) -> Result<Statement, ParserError> {
-        let role_name = self.parse_identifier(false)?;
+        let role_name = self.parse_identifier()?;
 
         // [ IN DATABASE _`database_name`_ ]
         let in_database = if self.parse_keywords(&[Keyword::IN, Keyword::DATABASE]) {
@@ -137,7 +137,7 @@ impl Parser<'_> {
 
         let operation = if self.parse_keyword(Keyword::RENAME) {
             if self.parse_keyword(Keyword::TO) {
-                let role_name = self.parse_identifier(false)?;
+                let role_name = self.parse_identifier()?;
                 AlterRoleOperation::RenameRole { role_name }
             } else {
                 return self.expected("TO after RENAME", self.peek_token());

src/tokenizer.rs

@@ -1144,29 +1144,15 @@ impl<'a> Tokenizer<'a> {
 
                     // match one period
                     if let Some('.') = chars.peek() {
-                        // Check if this actually is a float point number
-                        let mut char_clone = chars.peekable.clone();
-                        char_clone.next();
-                        // Next char should be a digit, otherwise, it is not a float point number
-                        if char_clone
-                            .peek()
-                            .map(|c| c.is_ascii_digit())
-                            .unwrap_or(false)
-                        {
                         s.push('.');
                         chars.next();
-                        } else if !s.is_empty() {
+                    }
-                            // Number might be part of period separated construct. Keep the period for next token
+                    s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
-                            // e.g. a-12.b
+
-                            return Ok(Some(Token::Number(s, false)));
-                        } else {
                     // No number -> Token::Period
-                            chars.next();
+                    if s == "." {
                         return Ok(Some(Token::Period));
                     }
-                    }
-
-                    s += &peeking_take_while(chars, |ch| ch.is_ascii_digit());
 
                     let mut exponent_part = String::new();
                     // Parse exponent as number
@@ -2199,23 +2185,6 @@ mod tests {
         compare(expected, tokens);
     }
 
-    #[test]
-    fn tokenize_select_float_hyphenated_identifier() {
-        let sql = String::from("SELECT a-12.b");
-        let dialect = GenericDialect {};
-        let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
-        let expected = vec![
-            Token::make_keyword("SELECT"),
-            Token::Whitespace(Whitespace::Space),
-            Token::make_word("a", None),
-            Token::Minus,
-            Token::Number(String::from("12"), false),
-            Token::Period,
-            Token::make_word("b", None),
-        ];
-        compare(expected, tokens);
-    }
-
     #[test]
     fn tokenize_clickhouse_double_equal() {
         let sql = String::from("SELECT foo=='1'");

tests/sqlparser_common.rs

@@ -2964,6 +2964,113 @@ fn test_compound_expr() {
     }
 }
 
+#[test]
+fn test_double_value() {
+    let dialects = all_dialects();
+    let test_cases = vec![
+        gen_number_case_with_sign("0."),
+        gen_number_case_with_sign("0.0"),
+        gen_number_case_with_sign("0000."),
+        gen_number_case_with_sign("0000.00"),
+        gen_number_case_with_sign(".0"),
+        gen_number_case_with_sign(".00"),
+        gen_number_case_with_sign("0e0"),
+        gen_number_case_with_sign("0e+0"),
+        gen_number_case_with_sign("0e-0"),
+        gen_number_case_with_sign("0.e-0"),
+        gen_number_case_with_sign("0.e+0"),
+        gen_number_case_with_sign(".0e-0"),
+        gen_number_case_with_sign(".0e+0"),
+        gen_number_case_with_sign("00.0e+0"),
+        gen_number_case_with_sign("00.0e-0"),
+    ];
+
+    for (input, expected) in test_cases {
+        for (i, expr) in input.iter().enumerate() {
+            if let Statement::Query(query) =
+                dialects.one_statement_parses_to(&format!("SELECT {}", expr), "")
+            {
+                if let SetExpr::Select(select) = *query.body {
+                    assert_eq!(expected[i], select.projection[0]);
+                } else {
+                    panic!("Expected a SELECT statement");
+                }
+            } else {
+                panic!("Expected a SELECT statement");
+            }
+        }
+    }
+}
+
+fn gen_number_case(value: &str) -> (Vec<String>, Vec<SelectItem>) {
+    let input = vec![
+        value.to_string(),
+        format!("{} col_alias", value),
+        format!("{} AS col_alias", value),
+    ];
+    let expected = vec![
+        SelectItem::UnnamedExpr(Expr::Value(number(value))),
+        SelectItem::ExprWithAlias {
+            expr: Expr::Value(number(value)),
+            alias: Ident::new("col_alias"),
+        },
+        SelectItem::ExprWithAlias {
+            expr: Expr::Value(number(value)),
+            alias: Ident::new("col_alias"),
+        },
+    ];
+    (input, expected)
+}
+
+fn gen_sign_number_case(value: &str, op: UnaryOperator) -> (Vec<String>, Vec<SelectItem>) {
+    match op {
+        UnaryOperator::Plus | UnaryOperator::Minus => {}
+        _ => panic!("Invalid sign"),
+    }
+
+    let input = vec![
+        format!("{}{}", op, value),
+        format!("{}{} col_alias", op, value),
+        format!("{}{} AS col_alias", op, value),
+    ];
+    let expected = vec![
+        SelectItem::UnnamedExpr(Expr::UnaryOp {
+            op,
+            expr: Box::new(Expr::Value(number(value))),
+        }),
+        SelectItem::ExprWithAlias {
+            expr: Expr::UnaryOp {
+                op,
+                expr: Box::new(Expr::Value(number(value))),
+            },
+            alias: Ident::new("col_alias"),
+        },
+        SelectItem::ExprWithAlias {
+            expr: Expr::UnaryOp {
+                op,
+                expr: Box::new(Expr::Value(number(value))),
+            },
+            alias: Ident::new("col_alias"),
+        },
+    ];
+    (input, expected)
+}
+
+/// generate the test cases for signed and unsigned numbers
+/// For example, given "0.0", the test cases will be:
+/// - "0.0"
+/// - "+0.0"
+/// - "-0.0"
+fn gen_number_case_with_sign(number: &str) -> (Vec<String>, Vec<SelectItem>) {
+    let (mut input, mut expected) = gen_number_case(number);
+    for op in [UnaryOperator::Plus, UnaryOperator::Minus] {
+        let (input_sign, expected_sign) = gen_sign_number_case(number, op);
+        input.extend(input_sign);
+        expected.extend(expected_sign);
+    }
+    (input, expected)
+}
+
 #[test]
 fn parse_negative_value() {
     let sql1 = "SELECT -1";
@@ -12470,6 +12577,41 @@ fn parse_composite_access_expr() {
     all_dialects_where(|d| d.supports_struct_literal()).verified_stmt(
         "SELECT * FROM t WHERE STRUCT(STRUCT(1 AS a, NULL AS b) AS c, NULL AS d).c.a IS NOT NULL",
     );
+    let support_struct = all_dialects_where(|d| d.supports_struct_literal());
+    let stmt = support_struct
+        .verified_only_select("SELECT STRUCT(STRUCT(1 AS a, NULL AS b) AS c, NULL AS d).c.a");
+    let expected = SelectItem::UnnamedExpr(Expr::CompoundFieldAccess {
+        root: Box::new(Expr::Struct {
+            values: vec![
+                Expr::Named {
+                    name: Ident::new("c"),
+                    expr: Box::new(Expr::Struct {
+                        values: vec![
+                            Expr::Named {
+                                name: Ident::new("a"),
+                                expr: Box::new(Expr::Value(Number("1".parse().unwrap(), false))),
+                            },
+                            Expr::Named {
+                                name: Ident::new("b"),
+                                expr: Box::new(Expr::Value(Value::Null)),
+                            },
+                        ],
+                        fields: vec![],
+                    }),
+                },
+                Expr::Named {
+                    name: Ident::new("d"),
+                    expr: Box::new(Expr::Value(Value::Null)),
+                },
+            ],
+            fields: vec![],
+        }),
+        access_chain: vec![
+            AccessExpr::Dot(Expr::Identifier(Ident::new("c"))),
+            AccessExpr::Dot(Expr::Identifier(Ident::new("a"))),
+        ],
+    });
+    assert_eq!(stmt.projection[0], expected);
 }
 
 #[test]