Fix parse error on some prepared statement placeholders (#604)

sqlparser can now parse all the prepared statement placeholders supported by SQLite:

 - ?
 - ?NNN
 - @VVV
 - :VVV
 - $VVV

See: https://www.sqlite.org/lang_expr.html#varparam

This does not break existing support for postgresql's '@' operator

Fixes #603

src/parser.rs

@@ -570,7 +570,7 @@ impl<'a> Parser<'a> {
                     })
                 }
             }
-            Token::Placeholder(_) => {
+            Token::Placeholder(_) | Token::Colon | Token::AtSign => {
                 self.prev_token();
                 Ok(Expr::Value(self.parse_value()?))
             }
@@ -1774,7 +1774,7 @@ impl<'a> Parser<'a> {
                             .iter()
                             .any(|d| kw.keyword == *d) =>
                     {
-                        break
+                        break;
                     }
                     Token::RParen | Token::EOF => break,
                     _ => continue,
@@ -3038,6 +3038,11 @@ impl<'a> Parser<'a> {
             Token::EscapedStringLiteral(ref s) => Ok(Value::EscapedStringLiteral(s.to_string())),
             Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
             Token::Placeholder(ref s) => Ok(Value::Placeholder(s.to_string())),
+            tok @ Token::Colon | tok @ Token::AtSign => {
+                let ident = self.parse_identifier()?;
+                let placeholder = tok.to_string() + &ident.value;
+                Ok(Value::Placeholder(placeholder))
+            }
             unexpected => self.expected("a value", unexpected),
         }
     }
@@ -4892,12 +4897,12 @@ impl<'a> Parser<'a> {
                     Some(_) => {
                         return Err(ParserError::ParserError(
                             "expected UPDATE, DELETE or INSERT in merge clause".to_string(),
-                        ))
+                        ));
                     }
                     None => {
                         return Err(ParserError::ParserError(
                             "expected UPDATE, DELETE or INSERT in merge clause".to_string(),
-                        ))
+                        ));
                     }
                 },
             );

src/tokenizer.rs

@@ -678,13 +678,14 @@ impl<'a> Tokenizer<'a> {
                     }
                 }
                 '@' => self.consume_and_return(chars, Token::AtSign),
-                '?' => self.consume_and_return(chars, Token::Placeholder(String::from("?"))),
+                '?' => {
+                    chars.next();
+                    let s = peeking_take_while(chars, |ch| ch.is_numeric());
+                    Ok(Some(Token::Placeholder(String::from("?") + &s)))
+                }
                 '$' => {
                     chars.next();
-                    let s = peeking_take_while(
+                    let s = peeking_take_while(chars, |ch| ch.is_alphanumeric() || ch == '_');
-                        chars,
-                        |ch| matches!(ch, '0'..='9' | 'A'..='Z' | 'a'..='z'),
-                    );
                     Ok(Some(Token::Placeholder(String::from("$") + &s)))
                 }
                 //whitespace check (including unicode chars) should be last as it covers some of the chars above

tests/sqlparser_common.rs

@@ -22,6 +22,7 @@
 mod test_utils;
 
 use matches::assert_matches;
+use sqlparser::ast::SelectItem::UnnamedExpr;
 use sqlparser::ast::*;
 use sqlparser::dialect::{
     AnsiDialect, BigQueryDialect, ClickHouseDialect, GenericDialect, HiveDialect, MsSqlDialect,
@@ -5299,6 +5300,17 @@ fn test_placeholder() {
             rows: OffsetRows::None,
         }),
     );
+
+    let sql = "SELECT $fromage_français, :x, ?123";
+    let ast = dialects.verified_only_select(sql);
+    assert_eq!(
+        ast.projection,
+        vec![
+            UnnamedExpr(Expr::Value(Value::Placeholder("$fromage_français".into()))),
+            UnnamedExpr(Expr::Value(Value::Placeholder(":x".into()))),
+            UnnamedExpr(Expr::Value(Value::Placeholder("?123".into()))),
+        ]
+    );
 }
 
 #[test]

tests/sqlparser_sqlite.rs

@@ -16,8 +16,10 @@
 
 #[macro_use]
 mod test_utils;
+
 use test_utils::*;
 
+use sqlparser::ast::SelectItem::UnnamedExpr;
 use sqlparser::ast::*;
 use sqlparser::dialect::{GenericDialect, SQLiteDialect};
 use sqlparser::tokenizer::Token;
@@ -73,14 +75,14 @@ fn parse_create_table_auto_increment() {
                     options: vec![
                         ColumnOptionDef {
                             name: None,
-                            option: ColumnOption::Unique { is_primary: true }
+                            option: ColumnOption::Unique { is_primary: true },
                         },
                         ColumnOptionDef {
                             name: None,
                             option: ColumnOption::DialectSpecific(vec![Token::make_keyword(
                                 "AUTOINCREMENT"
-                            )])
+                            )]),
-                        }
+                        },
                     ],
                 }],
                 columns
@@ -118,6 +120,19 @@ fn parse_create_sqlite_quote() {
     }
 }
 
+#[test]
+fn test_placeholder() {
+    // In postgres, this would be the absolute value operator '@' applied to the column 'xxx'
+    // But in sqlite, this is a named parameter.
+    // see https://www.sqlite.org/lang_expr.html#varparam
+    let sql = "SELECT @xxx";
+    let ast = sqlite().verified_only_select(sql);
+    assert_eq!(
+        ast.projection[0],
+        UnnamedExpr(Expr::Value(Value::Placeholder("@xxx".into()))),
+    );
+}
+
 fn sqlite() -> TestedDialects {
     TestedDialects {
         dialects: vec![Box::new(SQLiteDialect {})],