mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-12-23 11:12:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge 0f17b327b9 into c6933bb8ec

Browse source
This commit is contained in:
eyalleshem 2025-12-18 17:55:45 +02:00 • committed by GitHub
commit 2d30e4fc25
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1724 additions and 1409 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/ast/helpers/attached_token.rs
View file

@ -80,7 +80,7 @@ use sqlparser_derive::{Visit, VisitMut};
#[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct AttachedToken(pub TokenWithSpan);
pub struct AttachedToken(pub TokenWithSpan<'static>);
impl AttachedToken {
/// Return a new Empty AttachedToken
@ -123,13 +123,13 @@ impl Hash for AttachedToken {
}
}
impl From<TokenWithSpan> for AttachedToken {
fn from(value: TokenWithSpan) -> Self {
impl From<TokenWithSpan<'static>> for AttachedToken {
fn from(value: TokenWithSpan<'static>) -> Self {
AttachedToken(value)
}
}
impl From<AttachedToken> for TokenWithSpan {
impl From<AttachedToken> for TokenWithSpan<'static> {
fn from(value: AttachedToken) -> Self {
value.0
}

26
src/ast/visitor.rs
View file

@ -17,6 +17,11 @@
//! Recursive visitors for ast Nodes. See [`Visitor`] for more details.
#[cfg(not(feature = "std"))]
use alloc::borrow::Cow;
#[cfg(feature = "std")]
use std::borrow::Cow;
use crate::ast::{Expr, ObjectName, Query, Statement, TableFactor, Value};
use core::ops::ControlFlow;
@ -118,6 +123,19 @@ visit_noop!(u8, u16, u32, u64, i8, i16, i32, i64, char, bool, String);
#[cfg(feature = "bigdecimal")]
visit_noop!(bigdecimal::BigDecimal);
// Implement Visit and VisitMut for Cow<str> to support the lifetime parameter in BorrowedToken
impl<'a> Visit for Cow<'a, str> {
fn visit<V: Visitor>(&self, _visitor: &mut V) -> ControlFlow<V::Break> {
ControlFlow::Continue(())
}
}
impl<'a> VisitMut for Cow<'a, str> {
fn visit<V: VisitorMut>(&mut self, _visitor: &mut V) -> ControlFlow<V::Break> {
ControlFlow::Continue(())
}
}
/// A visitor that can be used to walk an AST tree.
///
/// `pre_visit_` methods are invoked before visiting all children of the
@ -751,7 +769,7 @@ mod tests {
fn do_visit<V: Visitor<Break = ()>>(sql: &str, visitor: &mut V) -> Statement {
let dialect = GenericDialect {};
let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
let tokens = Tokenizer::new(&dialect, sql).tokenized_owned().unwrap();
let s = Parser::new(&dialect)
.with_tokens(tokens)
.parse_statement()
@ -942,7 +960,9 @@ mod tests {
let sql = format!("SELECT x where {cond}");
let dialect = GenericDialect {};
let tokens = Tokenizer::new(&dialect, sql.as_str()).tokenize().unwrap();
let tokens = Tokenizer::new(&dialect, sql.as_str())
.tokenized_owned()
.unwrap();
let s = Parser::new(&dialect)
.with_tokens(tokens)
.parse_statement()
@ -983,7 +1003,7 @@ mod visit_mut_tests {
fn do_visit_mut<V: VisitorMut<Break = ()>>(sql: &str, visitor: &mut V) -> Statement {
let dialect = GenericDialect {};
let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
let tokens = Tokenizer::new(&dialect, sql).tokenized_owned().unwrap();
let mut s = Parser::new(&dialect)
.with_tokens(tokens)
.parse_statement()

208
src/dialect/mod.rs
View file

@ -53,7 +53,7 @@ use crate::ast::{ColumnOption, Expr, GranteesType, Ident, ObjectNamePart, Statem
pub use crate::keywords;
use crate::keywords::Keyword;
use crate::parser::{Parser, ParserError};
use crate::tokenizer::Token;
use crate::tokenizer::BorrowedToken;
#[cfg(not(feature = "std"))]
use alloc::boxed::Box;
@ -655,16 +655,16 @@ pub trait Dialect: Debug + Any {
let token = parser.peek_token();
debug!("get_next_precedence_full() {token:?}");
match token.token {
Token::Word(w) if w.keyword == Keyword::OR => Ok(p!(Or)),
Token::Word(w) if w.keyword == Keyword::AND => Ok(p!(And)),
Token::Word(w) if w.keyword == Keyword::XOR => Ok(p!(Xor)),
BorrowedToken::Word(w) if w.keyword == Keyword::OR => Ok(p!(Or)),
BorrowedToken::Word(w) if w.keyword == Keyword::AND => Ok(p!(And)),
BorrowedToken::Word(w) if w.keyword == Keyword::XOR => Ok(p!(Xor)),
Token::Word(w) if w.keyword == Keyword::AT => {
BorrowedToken::Word(w) if w.keyword == Keyword::AT => {
match (
parser.peek_nth_token(1).token,
parser.peek_nth_token(2).token,
) {
(Token::Word(w), Token::Word(w2))
(BorrowedToken::Word(w), BorrowedToken::Word(w2))
if w.keyword == Keyword::TIME && w2.keyword == Keyword::ZONE =>
{
Ok(p!(AtTz))
@ -673,102 +673,112 @@ pub trait Dialect: Debug + Any {
}
}
Token::Word(w) if w.keyword == Keyword::NOT => match parser.peek_nth_token(1).token {
// The precedence of NOT varies depending on keyword that
// follows it. If it is followed by IN, BETWEEN, or LIKE,
// it takes on the precedence of those tokens. Otherwise, it
// is not an infix operator, and therefore has zero
// precedence.
Token::Word(w) if w.keyword == Keyword::IN => Ok(p!(Between)),
Token::Word(w) if w.keyword == Keyword::BETWEEN => Ok(p!(Between)),
Token::Word(w) if w.keyword == Keyword::LIKE => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::ILIKE => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::RLIKE => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::REGEXP => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::MATCH => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::SIMILAR => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::MEMBER => Ok(p!(Like)),
Token::Word(w)
if w.keyword == Keyword::NULL && !parser.in_column_definition_state() =>
{
Ok(p!(Is))
BorrowedToken::Word(w) if w.keyword == Keyword::NOT => {
match parser.peek_nth_token(1).token {
// The precedence of NOT varies depending on keyword that
// follows it. If it is followed by IN, BETWEEN, or LIKE,
// it takes on the precedence of those tokens. Otherwise, it
// is not an infix operator, and therefore has zero
// precedence.
BorrowedToken::Word(w) if w.keyword == Keyword::IN => Ok(p!(Between)),
BorrowedToken::Word(w) if w.keyword == Keyword::BETWEEN => Ok(p!(Between)),
BorrowedToken::Word(w) if w.keyword == Keyword::LIKE => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::ILIKE => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::RLIKE => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::REGEXP => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::MATCH => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::SIMILAR => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::MEMBER => Ok(p!(Like)),
BorrowedToken::Word(w)
if w.keyword == Keyword::NULL && !parser.in_column_definition_state() =>
{
Ok(p!(Is))
}
_ => Ok(self.prec_unknown()),
}
_ => Ok(self.prec_unknown()),
},
Token::Word(w) if w.keyword == Keyword::NOTNULL && self.supports_notnull_operator() => {
}
BorrowedToken::Word(w)
if w.keyword == Keyword::NOTNULL && self.supports_notnull_operator() =>
{
Ok(p!(Is))
}
Token::Word(w) if w.keyword == Keyword::IS => Ok(p!(Is)),
Token::Word(w) if w.keyword == Keyword::IN => Ok(p!(Between)),
Token::Word(w) if w.keyword == Keyword::BETWEEN => Ok(p!(Between)),
Token::Word(w) if w.keyword == Keyword::OVERLAPS => Ok(p!(Between)),
Token::Word(w) if w.keyword == Keyword::LIKE => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::ILIKE => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::RLIKE => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::REGEXP => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::MATCH => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::SIMILAR => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::MEMBER => Ok(p!(Like)),
Token::Word(w) if w.keyword == Keyword::OPERATOR => Ok(p!(Between)),
Token::Word(w) if w.keyword == Keyword::DIV => Ok(p!(MulDivModOp)),
Token::Period => Ok(p!(Period)),
Token::Assignment
| Token::Eq
| Token::Lt
| Token::LtEq
| Token::Neq
| Token::Gt
| Token::GtEq
| Token::DoubleEq
| Token::Tilde
| Token::TildeAsterisk
| Token::ExclamationMarkTilde
| Token::ExclamationMarkTildeAsterisk
| Token::DoubleTilde
| Token::DoubleTildeAsterisk
| Token::ExclamationMarkDoubleTilde
| Token::ExclamationMarkDoubleTildeAsterisk
| Token::Spaceship => Ok(p!(Eq)),
Token::Pipe
| Token::QuestionMarkDash
| Token::DoubleSharp
| Token::Overlap
| Token::AmpersandLeftAngleBracket
| Token::AmpersandRightAngleBracket
| Token::QuestionMarkDashVerticalBar
| Token::AmpersandLeftAngleBracketVerticalBar
| Token::VerticalBarAmpersandRightAngleBracket
| Token::TwoWayArrow
| Token::LeftAngleBracketCaret
| Token::RightAngleBracketCaret
| Token::QuestionMarkSharp
| Token::QuestionMarkDoubleVerticalBar
| Token::QuestionPipe
| Token::TildeEqual
| Token::AtSign
| Token::ShiftLeftVerticalBar
| Token::VerticalBarShiftRight => Ok(p!(Pipe)),
Token::Caret | Token::Sharp | Token::ShiftRight | Token::ShiftLeft => Ok(p!(Caret)),
Token::Ampersand => Ok(p!(Ampersand)),
Token::Plus | Token::Minus => Ok(p!(PlusMinus)),
Token::Mul | Token::Div | Token::DuckIntDiv | Token::Mod | Token::StringConcat => {
Ok(p!(MulDivModOp))
}
Token::DoubleColon | Token::ExclamationMark | Token::LBracket | Token::CaretAt => {
Ok(p!(DoubleColon))
}
Token::Arrow
| Token::LongArrow
| Token::HashArrow
| Token::HashLongArrow
| Token::AtArrow
| Token::ArrowAt
| Token::HashMinus
| Token::AtQuestion
| Token::AtAt
| Token::Question
| Token::QuestionAnd
| Token::CustomBinaryOperator(_) => Ok(p!(PgOther)),
BorrowedToken::Word(w) if w.keyword == Keyword::IS => Ok(p!(Is)),
BorrowedToken::Word(w) if w.keyword == Keyword::IN => Ok(p!(Between)),
BorrowedToken::Word(w) if w.keyword == Keyword::BETWEEN => Ok(p!(Between)),
BorrowedToken::Word(w) if w.keyword == Keyword::OVERLAPS => Ok(p!(Between)),
BorrowedToken::Word(w) if w.keyword == Keyword::LIKE => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::ILIKE => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::RLIKE => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::REGEXP => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::MATCH => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::SIMILAR => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::MEMBER => Ok(p!(Like)),
BorrowedToken::Word(w) if w.keyword == Keyword::OPERATOR => Ok(p!(Between)),
BorrowedToken::Word(w) if w.keyword == Keyword::DIV => Ok(p!(MulDivModOp)),
BorrowedToken::Period => Ok(p!(Period)),
BorrowedToken::Assignment
| BorrowedToken::Eq
| BorrowedToken::Lt
| BorrowedToken::LtEq
| BorrowedToken::Neq
| BorrowedToken::Gt
| BorrowedToken::GtEq
| BorrowedToken::DoubleEq
| BorrowedToken::Tilde
| BorrowedToken::TildeAsterisk
| BorrowedToken::ExclamationMarkTilde
| BorrowedToken::ExclamationMarkTildeAsterisk
| BorrowedToken::DoubleTilde
| BorrowedToken::DoubleTildeAsterisk
| BorrowedToken::ExclamationMarkDoubleTilde
| BorrowedToken::ExclamationMarkDoubleTildeAsterisk
| BorrowedToken::Spaceship => Ok(p!(Eq)),
BorrowedToken::Pipe
| BorrowedToken::QuestionMarkDash
| BorrowedToken::DoubleSharp
| BorrowedToken::Overlap
| BorrowedToken::AmpersandLeftAngleBracket
| BorrowedToken::AmpersandRightAngleBracket
| BorrowedToken::QuestionMarkDashVerticalBar
| BorrowedToken::AmpersandLeftAngleBracketVerticalBar
| BorrowedToken::VerticalBarAmpersandRightAngleBracket
| BorrowedToken::TwoWayArrow
| BorrowedToken::LeftAngleBracketCaret
| BorrowedToken::RightAngleBracketCaret
| BorrowedToken::QuestionMarkSharp
| BorrowedToken::QuestionMarkDoubleVerticalBar
| BorrowedToken::QuestionPipe
| BorrowedToken::TildeEqual
| BorrowedToken::AtSign
| BorrowedToken::ShiftLeftVerticalBar
| BorrowedToken::VerticalBarShiftRight => Ok(p!(Pipe)),
BorrowedToken::Caret
| BorrowedToken::Sharp
| BorrowedToken::ShiftRight
| BorrowedToken::ShiftLeft => Ok(p!(Caret)),
BorrowedToken::Ampersand => Ok(p!(Ampersand)),
BorrowedToken::Plus | BorrowedToken::Minus => Ok(p!(PlusMinus)),
BorrowedToken::Mul
| BorrowedToken::Div
| BorrowedToken::DuckIntDiv
| BorrowedToken::Mod
| BorrowedToken::StringConcat => Ok(p!(MulDivModOp)),
BorrowedToken::DoubleColon
| BorrowedToken::ExclamationMark
| BorrowedToken::LBracket
| BorrowedToken::CaretAt => Ok(p!(DoubleColon)),
BorrowedToken::Arrow
| BorrowedToken::LongArrow
| BorrowedToken::HashArrow
| BorrowedToken::HashLongArrow
| BorrowedToken::AtArrow
| BorrowedToken::ArrowAt
| BorrowedToken::HashMinus
| BorrowedToken::AtQuestion
| BorrowedToken::AtAt
| BorrowedToken::Question
| BorrowedToken::QuestionAnd
| BorrowedToken::CustomBinaryOperator(_) => Ok(p!(PgOther)),
_ => Ok(self.prec_unknown()),
}
}

25
src/dialect/mssql.rs
View file

@ -23,7 +23,7 @@ use crate::ast::{
use crate::dialect::Dialect;
use crate::keywords::{self, Keyword};
use crate::parser::{Parser, ParserError};
use crate::tokenizer::Token;
use crate::tokenizer::BorrowedToken;
#[cfg(not(feature = "std"))]
use alloc::{vec, vec::Vec};
@ -167,19 +167,19 @@ impl MsSqlDialect {
let statements = self.parse_statement_list(parser, Some(Keyword::END))?;
let end_token = parser.expect_keyword(Keyword::END)?;
ConditionalStatementBlock {
start_token: AttachedToken(if_token),
start_token: AttachedToken(if_token.to_static()),
condition: Some(condition),
then_token: None,
conditional_statements: ConditionalStatements::BeginEnd(BeginEndStatements {
begin_token: AttachedToken(begin_token),
begin_token: AttachedToken(begin_token.to_static()),
statements,
end_token: AttachedToken(end_token),
end_token: AttachedToken(end_token.to_static()),
}),
}
} else {
let stmt = parser.parse_statement()?;
ConditionalStatementBlock {
start_token: AttachedToken(if_token),
start_token: AttachedToken(if_token.to_static()),
condition: Some(condition),
then_token: None,
conditional_statements: ConditionalStatements::Sequence {
@ -189,7 +189,8 @@ impl MsSqlDialect {
};
let mut prior_statement_ended_with_semi_colon = false;
while let Token::SemiColon = parser.peek_token_ref().token {
while let BorrowedToken::SemiColon = parser.peek_token_ref().token {
parser.advance_token();
prior_statement_ended_with_semi_colon = true;
}
@ -202,19 +203,19 @@ impl MsSqlDialect {
let statements = self.parse_statement_list(parser, Some(Keyword::END))?;
let end_token = parser.expect_keyword(Keyword::END)?;
else_block = Some(ConditionalStatementBlock {
start_token: AttachedToken(else_token),
start_token: AttachedToken(else_token.to_static()),
condition: None,
then_token: None,
conditional_statements: ConditionalStatements::BeginEnd(BeginEndStatements {
begin_token: AttachedToken(begin_token),
begin_token: AttachedToken(begin_token.to_static()),
statements,
end_token: AttachedToken(end_token),
end_token: AttachedToken(end_token.to_static()),
}),
});
} else {
let stmt = parser.parse_statement()?;
else_block = Some(ConditionalStatementBlock {
start_token: AttachedToken(else_token),
start_token: AttachedToken(else_token.to_static()),
condition: None,
then_token: None,
conditional_statements: ConditionalStatements::Sequence {
@ -284,7 +285,7 @@ impl MsSqlDialect {
) -> Result<Vec<Statement>, ParserError> {
let mut stmts = Vec::new();
loop {
if let Token::EOF = parser.peek_token_ref().token {
if let BorrowedToken::EOF = parser.peek_token_ref().token {
break;
}
if let Some(term) = terminal_keyword {
@ -293,7 +294,7 @@ impl MsSqlDialect {
}
}
stmts.push(parser.parse_statement()?);
while let Token::SemiColon = parser.peek_token_ref().token {
while let BorrowedToken::SemiColon = parser.peek_token_ref().token {
parser.advance_token();
}
}

46
src/dialect/postgresql.rs
View file

@ -31,7 +31,7 @@ use log::debug;
use crate::dialect::{Dialect, Precedence};
use crate::keywords::Keyword;
use crate::parser::{Parser, ParserError};
use crate::tokenizer::Token;
use crate::tokenizer::BorrowedToken;
/// A [`Dialect`] for [PostgreSQL](https://www.postgresql.org/)
#[derive(Debug)]
@ -110,32 +110,32 @@ impl Dialect for PostgreSqlDialect {
// we only return some custom value here when the behaviour (not merely the numeric value) differs
// from the default implementation
match token.token {
Token::Word(w)
BorrowedToken::Word(w)
if w.keyword == Keyword::COLLATE && !parser.in_column_definition_state() =>
{
Some(Ok(COLLATE_PREC))
}
Token::LBracket => Some(Ok(BRACKET_PREC)),
Token::Arrow
| Token::LongArrow
| Token::HashArrow
| Token::HashLongArrow
| Token::AtArrow
| Token::ArrowAt
| Token::HashMinus
| Token::AtQuestion
| Token::AtAt
| Token::Question
| Token::QuestionAnd
| Token::QuestionPipe
| Token::ExclamationMark
| Token::Overlap
| Token::CaretAt
| Token::StringConcat
| Token::Sharp
| Token::ShiftRight
| Token::ShiftLeft
| Token::CustomBinaryOperator(_) => Some(Ok(PG_OTHER_PREC)),
BorrowedToken::LBracket => Some(Ok(BRACKET_PREC)),
BorrowedToken::Arrow
| BorrowedToken::LongArrow
| BorrowedToken::HashArrow
| BorrowedToken::HashLongArrow
| BorrowedToken::AtArrow
| BorrowedToken::ArrowAt
| BorrowedToken::HashMinus
| BorrowedToken::AtQuestion
| BorrowedToken::AtAt
| BorrowedToken::Question
| BorrowedToken::QuestionAnd
| BorrowedToken::QuestionPipe
| BorrowedToken::ExclamationMark
| BorrowedToken::Overlap
| BorrowedToken::CaretAt
| BorrowedToken::StringConcat
| BorrowedToken::Sharp
| BorrowedToken::ShiftRight
| BorrowedToken::ShiftLeft
| BorrowedToken::CustomBinaryOperator(_) => Some(Ok(PG_OTHER_PREC)),
_ => None,
}
}

247
src/dialect/snowflake.rs
View file

@ -37,7 +37,7 @@ use crate::ast::{
use crate::dialect::{Dialect, Precedence};
use crate::keywords::Keyword;
use crate::parser::{IsOptional, Parser, ParserError};
use crate::tokenizer::Token;
use crate::tokenizer::BorrowedToken;
#[cfg(not(feature = "std"))]
use alloc::boxed::Box;
#[cfg(not(feature = "std"))]
@ -350,7 +350,7 @@ impl Dialect for SnowflakeDialect {
let token = parser.peek_token();
// Snowflake supports the `:` cast operator unlike other dialects
match token.token {
Token::Colon => Some(Ok(self.prec_value(Precedence::DoubleColon))),
BorrowedToken::Colon => Some(Ok(self.prec_value(Precedence::DoubleColon))),
_ => None,
}
}
@ -398,7 +398,7 @@ impl Dialect for SnowflakeDialect {
// e.g. `SELECT * EXCEPT (col1) FROM tbl`
Keyword::EXCEPT
// e.g. `INSERT INTO t SELECT 1 RETURNING *`
| Keyword::RETURNING if !matches!(parser.peek_token_ref().token, Token::Comma | Token::EOF) =>
| Keyword::RETURNING if !matches!(parser.peek_token_ref().token, BorrowedToken::Comma | BorrowedToken::EOF) =>
{
false
}
@ -448,7 +448,10 @@ impl Dialect for SnowflakeDialect {
| Keyword::UNPIVOT
| Keyword::EXCEPT
| Keyword::MATCH_RECOGNIZE
if !matches!(parser.peek_token_ref().token, Token::SemiColon | Token::EOF) =>
if !matches!(
parser.peek_token_ref().token,
BorrowedToken::SemiColon | BorrowedToken::EOF
) =>
{
false
}
@ -525,7 +528,9 @@ impl Dialect for SnowflakeDialect {
match kw {
Keyword::LIMIT if peek_for_limit_options(parser) => false,
// Table function
Keyword::TABLE if matches!(parser.peek_token_ref().token, Token::LParen) => true,
Keyword::TABLE if matches!(parser.peek_token_ref().token, BorrowedToken::LParen) => {
true
}
_ => !RESERVED_KEYWORDS_FOR_TABLE_FACTOR.contains(kw),
}
}
@ -583,10 +588,12 @@ impl Dialect for SnowflakeDialect {
// a LIMIT/FETCH keyword.
fn peek_for_limit_options(parser: &Parser) -> bool {
match &parser.peek_token_ref().token {
Token::Number(_, _) | Token::Placeholder(_) => true,
Token::SingleQuotedString(val) if val.is_empty() => true,
Token::DollarQuotedString(DollarQuotedString { value, .. }) if value.is_empty() => true,
Token::Word(w) if w.keyword == Keyword::NULL => true,
BorrowedToken::Number(_, _) | BorrowedToken::Placeholder(_) => true,
BorrowedToken::SingleQuotedString(val) if val.is_empty() => true,
BorrowedToken::DollarQuotedString(DollarQuotedString { value, .. }) if value.is_empty() => {
true
}
BorrowedToken::Word(w) if w.keyword == Keyword::NULL => true,
_ => false,
}
}
@ -594,7 +601,7 @@ fn peek_for_limit_options(parser: &Parser) -> bool {
fn parse_file_staging_command(kw: Keyword, parser: &Parser) -> Result<Statement, ParserError> {
let stage = parse_snowflake_stage_name(parser)?;
let pattern = if parser.parse_keyword(Keyword::PATTERN) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
Some(parser.parse_literal_string()?)
} else {
None
@ -631,7 +638,7 @@ fn parse_alter_dynamic_table(parser: &Parser) -> Result<Statement, ParserError>
);
};
let end_token = if parser.peek_token_ref().token == Token::SemiColon {
let end_token = if parser.peek_token_ref().token == BorrowedToken::SemiColon {
parser.peek_token_ref().clone()
} else {
parser.get_current_token().clone()
@ -645,7 +652,7 @@ fn parse_alter_dynamic_table(parser: &Parser) -> Result<Statement, ParserError>
location: None,
on_cluster: None,
table_type: Some(AlterTableType::Dynamic),
end_token: AttachedToken(end_token),
end_token: AttachedToken(end_token.to_static()),
}))
}
@ -701,7 +708,7 @@ pub fn parse_create_table(
loop {
let next_token = parser.next_token();
match &next_token.token {
Token::Word(word) => match word.keyword {
BorrowedToken::Word(word) => match word.keyword {
Keyword::COPY => {
parser.expect_keyword_is(Keyword::GRANTS)?;
builder = builder.copy_grants(true);
@ -732,36 +739,36 @@ pub fn parse_create_table(
}
Keyword::CLUSTER => {
parser.expect_keyword_is(Keyword::BY)?;
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let cluster_by = Some(WrappedCollection::Parentheses(
parser.parse_comma_separated(|p| p.parse_expr())?,
));
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
builder = builder.cluster_by(cluster_by)
}
Keyword::ENABLE_SCHEMA_EVOLUTION => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.enable_schema_evolution(Some(parser.parse_boolean_string()?));
}
Keyword::CHANGE_TRACKING => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.change_tracking(Some(parser.parse_boolean_string()?));
}
Keyword::DATA_RETENTION_TIME_IN_DAYS => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let data_retention_time_in_days = parser.parse_literal_uint()?;
builder =
builder.data_retention_time_in_days(Some(data_retention_time_in_days));
}
Keyword::MAX_DATA_EXTENSION_TIME_IN_DAYS => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let max_data_extension_time_in_days = parser.parse_literal_uint()?;
builder = builder
.max_data_extension_time_in_days(Some(max_data_extension_time_in_days));
}
Keyword::DEFAULT_DDL_COLLATION => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let default_ddl_collation = parser.parse_literal_string()?;
builder = builder.default_ddl_collation(Some(default_ddl_collation));
}
@ -784,17 +791,17 @@ pub fn parse_create_table(
parser.expect_keywords(&[Keyword::ACCESS, Keyword::POLICY])?;
let policy = parser.parse_object_name(false)?;
parser.expect_keyword_is(Keyword::ON)?;
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let columns = parser.parse_comma_separated(|p| p.parse_identifier())?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
builder =
builder.with_row_access_policy(Some(RowAccessPolicy::new(policy, columns)))
}
Keyword::TAG => {
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let tags = parser.parse_comma_separated(Parser::parse_tag)?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
builder = builder.with_tags(Some(tags));
}
Keyword::ON if parser.parse_keyword(Keyword::COMMIT) => {
@ -802,23 +809,23 @@ pub fn parse_create_table(
builder = builder.on_commit(on_commit);
}
Keyword::EXTERNAL_VOLUME => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder.external_volume = Some(parser.parse_literal_string()?);
}
Keyword::CATALOG => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder.catalog = Some(parser.parse_literal_string()?);
}
Keyword::BASE_LOCATION => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder.base_location = Some(parser.parse_literal_string()?);
}
Keyword::CATALOG_SYNC => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder.catalog_sync = Some(parser.parse_literal_string()?);
}
Keyword::STORAGE_SERIALIZATION_POLICY => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder.storage_serialization_policy =
Some(parse_storage_serialization_policy(parser)?);
@ -827,12 +834,12 @@ pub fn parse_create_table(
builder = builder.if_not_exists(true);
}
Keyword::TARGET_LAG => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let target_lag = parser.parse_literal_string()?;
builder = builder.target_lag(Some(target_lag));
}
Keyword::WAREHOUSE => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let warehouse = parser.parse_identifier()?;
builder = builder.warehouse(Some(warehouse));
}
@ -842,7 +849,7 @@ pub fn parse_create_table(
builder = builder.version(version);
}
Keyword::REFRESH_MODE => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let refresh_mode = match parser.parse_one_of_keywords(&[
Keyword::AUTO,
Keyword::FULL,
@ -856,7 +863,7 @@ pub fn parse_create_table(
builder = builder.refresh_mode(refresh_mode);
}
Keyword::INITIALIZE => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let initialize = match parser
.parse_one_of_keywords(&[Keyword::ON_CREATE, Keyword::ON_SCHEDULE])
{
@ -873,15 +880,15 @@ pub fn parse_create_table(
return parser.expected("end of statement", next_token);
}
},
Token::LParen => {
BorrowedToken::LParen => {
parser.prev_token();
let (columns, constraints) = parser.parse_columns()?;
builder = builder.columns(columns).constraints(constraints);
}
Token::EOF => {
BorrowedToken::EOF => {
break;
}
Token::SemiColon => {
BorrowedToken::SemiColon => {
parser.prev_token();
break;
}
@ -925,58 +932,58 @@ pub fn parse_create_database(
loop {
let next_token = parser.next_token();
match &next_token.token {
Token::Word(word) => match word.keyword {
BorrowedToken::Word(word) => match word.keyword {
Keyword::CLONE => {
builder = builder.clone_clause(Some(parser.parse_object_name(false)?));
}
Keyword::DATA_RETENTION_TIME_IN_DAYS => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder =
builder.data_retention_time_in_days(Some(parser.parse_literal_uint()?));
}
Keyword::MAX_DATA_EXTENSION_TIME_IN_DAYS => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder =
builder.max_data_extension_time_in_days(Some(parser.parse_literal_uint()?));
}
Keyword::EXTERNAL_VOLUME => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.external_volume(Some(parser.parse_literal_string()?));
}
Keyword::CATALOG => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.catalog(Some(parser.parse_literal_string()?));
}
Keyword::REPLACE_INVALID_CHARACTERS => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder =
builder.replace_invalid_characters(Some(parser.parse_boolean_string()?));
}
Keyword::DEFAULT_DDL_COLLATION => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.default_ddl_collation(Some(parser.parse_literal_string()?));
}
Keyword::STORAGE_SERIALIZATION_POLICY => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let policy = parse_storage_serialization_policy(parser)?;
builder = builder.storage_serialization_policy(Some(policy));
}
Keyword::COMMENT => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.comment(Some(parser.parse_literal_string()?));
}
Keyword::CATALOG_SYNC => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.catalog_sync(Some(parser.parse_literal_string()?));
}
Keyword::CATALOG_SYNC_NAMESPACE_FLATTEN_DELIMITER => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
builder = builder.catalog_sync_namespace_flatten_delimiter(Some(
parser.parse_literal_string()?,
));
}
Keyword::CATALOG_SYNC_NAMESPACE_MODE => {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let mode =
match parser.parse_one_of_keywords(&[Keyword::NEST, Keyword::FLATTEN]) {
Some(Keyword::NEST) => CatalogSyncNamespaceMode::Nest,
@ -989,19 +996,19 @@ pub fn parse_create_database(
}
Keyword::WITH => {
if parser.parse_keyword(Keyword::TAG) {
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let tags = parser.parse_comma_separated(Parser::parse_tag)?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
builder = builder.with_tags(Some(tags));
} else if parser.parse_keyword(Keyword::CONTACT) {
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let contacts = parser.parse_comma_separated(|p| {
let purpose = p.parse_identifier()?.value;
p.expect_token(&Token::Eq)?;
p.expect_token(&BorrowedToken::Eq)?;
let contact = p.parse_identifier()?.value;
Ok(ContactEntry { purpose, contact })
})?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
builder = builder.with_contacts(Some(contacts));
} else {
return parser.expected("TAG or CONTACT", next_token);
@ -1009,7 +1016,7 @@ pub fn parse_create_database(
}
_ => return parser.expected("end of statement", next_token),
},
Token::SemiColon | Token::EOF => break,
BorrowedToken::SemiColon | BorrowedToken::EOF => break,
_ => return parser.expected("end of statement", next_token),
}
}
@ -1021,7 +1028,7 @@ pub fn parse_storage_serialization_policy(
) -> Result<StorageSerializationPolicy, ParserError> {
let next_token = parser.next_token();
match &next_token.token {
Token::Word(w) => match w.keyword {
BorrowedToken::Word(w) => match w.keyword {
Keyword::COMPATIBLE => Ok(StorageSerializationPolicy::Compatible),
Keyword::OPTIMIZED => Ok(StorageSerializationPolicy::Optimized),
_ => parser.expected("storage_serialization_policy", next_token),
@ -1048,25 +1055,25 @@ pub fn parse_create_stage(
// [ directoryTableParams ]
if parser.parse_keyword(Keyword::DIRECTORY) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
directory_table_params = parser.parse_key_value_options(true, &[])?.options;
}
// [ file_format]
if parser.parse_keyword(Keyword::FILE_FORMAT) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
file_format = parser.parse_key_value_options(true, &[])?.options;
}
// [ copy_options ]
if parser.parse_keyword(Keyword::COPY_OPTIONS) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
copy_options = parser.parse_key_value_options(true, &[])?.options;
}
// [ comment ]
if parser.parse_keyword(Keyword::COMMENT) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
comment = Some(parser.parse_comment_value()?);
}
@ -1096,23 +1103,23 @@ pub fn parse_stage_name_identifier(parser: &Parser) -> Result<Ident, ParserError
let mut ident = String::new();
while let Some(next_token) = parser.next_token_no_skip() {
match &next_token.token {
Token::Whitespace(_) | Token::SemiColon => break,
Token::Period => {
BorrowedToken::Whitespace(_) | BorrowedToken::SemiColon => break,
BorrowedToken::Period => {
parser.prev_token();
break;
}
Token::RParen => {
BorrowedToken::RParen => {
parser.prev_token();
break;
}
Token::AtSign => ident.push('@'),
Token::Tilde => ident.push('~'),
Token::Mod => ident.push('%'),
Token::Div => ident.push('/'),
Token::Plus => ident.push('+'),
Token::Minus => ident.push('-'),
Token::Number(n, _) => ident.push_str(n),
Token::Word(w) => ident.push_str(&w.to_string()),
BorrowedToken::AtSign => ident.push('@'),
BorrowedToken::Tilde => ident.push('~'),
BorrowedToken::Mod => ident.push('%'),
BorrowedToken::Div => ident.push('/'),
BorrowedToken::Plus => ident.push('+'),
BorrowedToken::Minus => ident.push('-'),
BorrowedToken::Number(n, _) => ident.push_str(n),
BorrowedToken::Word(w) => ident.push_str(&w.to_string()),
_ => return parser.expected("stage name identifier", parser.peek_token()),
}
}
@ -1121,12 +1128,12 @@ pub fn parse_stage_name_identifier(parser: &Parser) -> Result<Ident, ParserError
pub fn parse_snowflake_stage_name(parser: &Parser) -> Result<ObjectName, ParserError> {
match parser.next_token().token {
Token::AtSign => {
BorrowedToken::AtSign => {
parser.prev_token();
let mut idents = vec![];
loop {
idents.push(parse_stage_name_identifier(parser)?);
if !parser.consume_token(&Token::Period) {
if !parser.consume_token(&BorrowedToken::Period) {
break;
}
}
@ -1144,9 +1151,11 @@ pub fn parse_snowflake_stage_name(parser: &Parser) -> Result<ObjectName, ParserE
pub fn parse_copy_into(parser: &Parser) -> Result<Statement, ParserError> {
let kind = match parser.peek_token().token {
// Indicates an internal stage
Token::AtSign => CopyIntoSnowflakeKind::Location,
BorrowedToken::AtSign => CopyIntoSnowflakeKind::Location,
// Indicates an external stage, i.e. s3://, gcs:// or azure://
Token::SingleQuotedString(s) if s.contains("://") => CopyIntoSnowflakeKind::Location,
BorrowedToken::SingleQuotedString(s) if s.contains("://") => {
CopyIntoSnowflakeKind::Location
}
_ => CopyIntoSnowflakeKind::Table,
};
@ -1180,13 +1189,15 @@ pub fn parse_copy_into(parser: &Parser) -> Result<Statement, ParserError> {
}
let into_columns = match &parser.peek_token().token {
Token::LParen => Some(parser.parse_parenthesized_column_list(IsOptional::Optional, true)?),
BorrowedToken::LParen => {
Some(parser.parse_parenthesized_column_list(IsOptional::Optional, true)?)
}
_ => None,
};
parser.expect_keyword_is(Keyword::FROM)?;
match parser.next_token().token {
Token::LParen if kind == CopyIntoSnowflakeKind::Table => {
BorrowedToken::LParen if kind == CopyIntoSnowflakeKind::Table => {
// Data load with transformations
parser.expect_keyword_is(Keyword::SELECT)?;
from_transformations = parse_select_items_for_data_load(parser)?;
@ -1199,12 +1210,12 @@ pub fn parse_copy_into(parser: &Parser) -> Result<Statement, ParserError> {
from_stage_alias = parser
.maybe_parse_table_alias()?
.map(|table_alias| table_alias.name);
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
}
Token::LParen if kind == CopyIntoSnowflakeKind::Location => {
BorrowedToken::LParen if kind == CopyIntoSnowflakeKind::Location => {
// Data unload with a query
from_query = Some(parser.parse_query()?);
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
}
_ => {
parser.prev_token();
@ -1214,7 +1225,7 @@ pub fn parse_copy_into(parser: &Parser) -> Result<Statement, ParserError> {
// as
from_stage_alias = if parser.parse_keyword(Keyword::AS) {
Some(match parser.next_token().token {
Token::Word(w) => Ok(Ident::new(w.value)),
BorrowedToken::Word(w) => Ok(Ident::new(w.value)),
_ => parser.expected("stage alias", parser.peek_token()),
}?)
} else {
@ -1226,53 +1237,53 @@ pub fn parse_copy_into(parser: &Parser) -> Result<Statement, ParserError> {
loop {
// FILE_FORMAT
if parser.parse_keyword(Keyword::FILE_FORMAT) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
file_format = parser.parse_key_value_options(true, &[])?.options;
// PARTITION BY
} else if parser.parse_keywords(&[Keyword::PARTITION, Keyword::BY]) {
partition = Some(Box::new(parser.parse_expr()?))
// FILES
} else if parser.parse_keyword(Keyword::FILES) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::Eq)?;
parser.expect_token(&BorrowedToken::LParen)?;
let mut continue_loop = true;
while continue_loop {
continue_loop = false;
let next_token = parser.next_token();
match next_token.token {
Token::SingleQuotedString(s) => files.push(s),
BorrowedToken::SingleQuotedString(s) => files.push(s),
_ => parser.expected("file token", next_token)?,
};
if parser.next_token().token.eq(&Token::Comma) {
if parser.next_token().token.eq(&BorrowedToken::Comma) {
continue_loop = true;
} else {
parser.prev_token(); // not a comma, need to go back
}
}
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
// PATTERN
} else if parser.parse_keyword(Keyword::PATTERN) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
let next_token = parser.next_token();
pattern = Some(match next_token.token {
Token::SingleQuotedString(s) => s,
BorrowedToken::SingleQuotedString(s) => s,
_ => parser.expected("pattern", next_token)?,
});
// VALIDATION MODE
} else if parser.parse_keyword(Keyword::VALIDATION_MODE) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
validation_mode = Some(parser.next_token().token.to_string());
// COPY OPTIONS
} else if parser.parse_keyword(Keyword::COPY_OPTIONS) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
copy_options = parser.parse_key_value_options(true, &[])?.options;
} else {
match parser.next_token().token {
Token::SemiColon | Token::EOF => break,
Token::Comma => continue,
BorrowedToken::SemiColon | BorrowedToken::EOF => break,
BorrowedToken::Comma => continue,
// In `COPY INTO <location>` the copy options do not have a shared key
// like in `COPY INTO <table>`
Token::Word(key) => copy_options.push(parser.parse_key_value_option(&key)?),
BorrowedToken::Word(key) => copy_options.push(parser.parse_key_value_option(&key)?),
_ => return parser.expected("another copy option, ; or EOF'", parser.peek_token()),
}
}
@ -1315,7 +1326,7 @@ fn parse_select_items_for_data_load(
parser.parse_select_item()?,
)),
}
if matches!(parser.peek_token_ref().token, Token::Comma) {
if matches!(parser.peek_token_ref().token, BorrowedToken::Comma) {
parser.advance_token();
} else {
break;
@ -1332,13 +1343,13 @@ fn parse_select_item_for_data_load(parser: &Parser) -> Result<StageLoadSelectIte
let next_token = parser.next_token();
match next_token.token {
Token::Placeholder(w) => {
BorrowedToken::Placeholder(w) => {
file_col_num = w.to_string().split_off(1).parse::<i32>().map_err(|e| {
ParserError::ParserError(format!("Could not parse '{w}' as i32: {e}"))
})?;
Ok(())
}
Token::Word(w) => {
BorrowedToken::Word(w) => {
alias = Some(Ident::new(w.value));
Ok(())
}
@ -1346,11 +1357,11 @@ fn parse_select_item_for_data_load(parser: &Parser) -> Result<StageLoadSelectIte
}?;
if alias.is_some() {
parser.expect_token(&Token::Period)?;
parser.expect_token(&BorrowedToken::Period)?;
// now we get col_num token
let col_num_token = parser.next_token();
match col_num_token.token {
Token::Placeholder(w) => {
BorrowedToken::Placeholder(w) => {
file_col_num = w.to_string().split_off(1).parse::<i32>().map_err(|e| {
ParserError::ParserError(format!("Could not parse '{w}' as i32: {e}"))
})?;
@ -1362,10 +1373,10 @@ fn parse_select_item_for_data_load(parser: &Parser) -> Result<StageLoadSelectIte
// try extracting optional element
match parser.next_token().token {
Token::Colon => {
BorrowedToken::Colon => {
// parse element
element = Some(Ident::new(match parser.next_token().token {
Token::Word(w) => Ok(w.value),
BorrowedToken::Word(w) => Ok(w.value),
_ => parser.expected("file_col_num", parser.peek_token()),
}?));
}
@ -1378,7 +1389,7 @@ fn parse_select_item_for_data_load(parser: &Parser) -> Result<StageLoadSelectIte
// as
if parser.parse_keyword(Keyword::AS) {
item_as = Some(match parser.next_token().token {
Token::Word(w) => Ok(Ident::new(w.value)),
BorrowedToken::Word(w) => Ok(Ident::new(w.value)),
_ => parser.expected("column item alias", parser.peek_token()),
}?);
}
@ -1404,31 +1415,31 @@ fn parse_stage_params(parser: &Parser) -> Result<StageParamsObject, ParserError>
// URL
if parser.parse_keyword(Keyword::URL) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
url = Some(match parser.next_token().token {
Token::SingleQuotedString(word) => Ok(word),
BorrowedToken::SingleQuotedString(word) => Ok(word),
_ => parser.expected("a URL statement", parser.peek_token()),
}?)
}
// STORAGE INTEGRATION
if parser.parse_keyword(Keyword::STORAGE_INTEGRATION) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
storage_integration = Some(parser.next_token().token.to_string());
}
// ENDPOINT
if parser.parse_keyword(Keyword::ENDPOINT) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
endpoint = Some(match parser.next_token().token {
Token::SingleQuotedString(word) => Ok(word),
BorrowedToken::SingleQuotedString(word) => Ok(word),
_ => parser.expected("an endpoint statement", parser.peek_token()),
}?)
}
// CREDENTIALS
if parser.parse_keyword(Keyword::CREDENTIALS) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
credentials = KeyValueOptions {
options: parser.parse_key_value_options(true, &[])?.options,
delimiter: KeyValueOptionsDelimiter::Space,
@ -1437,7 +1448,7 @@ fn parse_stage_params(parser: &Parser) -> Result<StageParamsObject, ParserError>
// ENCRYPTION
if parser.parse_keyword(Keyword::ENCRYPTION) {
parser.expect_token(&Token::Eq)?;
parser.expect_token(&BorrowedToken::Eq)?;
encryption = KeyValueOptions {
options: parser.parse_key_value_options(true, &[])?.options,
delimiter: KeyValueOptionsDelimiter::Space,
@ -1463,12 +1474,12 @@ fn parse_session_options(parser: &Parser, set: bool) -> Result<Vec<KeyValueOptio
loop {
let next_token = parser.peek_token();
match next_token.token {
Token::SemiColon | Token::EOF => break,
Token::Comma => {
BorrowedToken::SemiColon | BorrowedToken::EOF => break,
BorrowedToken::Comma => {
parser.advance_token();
continue;
}
Token::Word(key) => {
BorrowedToken::Word(key) => {
parser.advance_token();
if set {
let option = parser.parse_key_value_option(&key)?;
@ -1501,11 +1512,11 @@ fn parse_session_options(parser: &Parser, set: bool) -> Result<Vec<KeyValueOptio
/// ```
/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table
fn parse_identity_property(parser: &Parser) -> Result<IdentityProperty, ParserError> {
let parameters = if parser.consume_token(&Token::LParen) {
let parameters = if parser.consume_token(&BorrowedToken::LParen) {
let seed = parser.parse_number()?;
parser.expect_token(&Token::Comma)?;
parser.expect_token(&BorrowedToken::Comma)?;
let increment = parser.parse_number()?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
Some(IdentityPropertyFormatKind::FunctionCall(
IdentityParameters { seed, increment },
@ -1541,9 +1552,9 @@ fn parse_column_policy_property(
) -> Result<ColumnPolicyProperty, ParserError> {
let policy_name = parser.parse_object_name(false)?;
let using_columns = if parser.parse_keyword(Keyword::USING) {
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let columns = parser.parse_comma_separated(|p| p.parse_identifier())?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
Some(columns)
} else {
None
@ -1563,9 +1574,9 @@ fn parse_column_policy_property(
/// ```
/// [Snowflake]: https://docs.snowflake.com/en/sql-reference/sql/create-table
fn parse_column_tags(parser: &Parser, with: bool) -> Result<TagsColumnOption, ParserError> {
parser.expect_token(&Token::LParen)?;
parser.expect_token(&BorrowedToken::LParen)?;
let tags = parser.parse_comma_separated(Parser::parse_tag)?;
parser.expect_token(&Token::RParen)?;
parser.expect_token(&BorrowedToken::RParen)?;
Ok(TagsColumnOption { with, tags })
}

16
src/parser/alter.rs
View file

@ -26,7 +26,7 @@ use crate::{
},
dialect::{MsSqlDialect, PostgreSqlDialect},
keywords::Keyword,
tokenizer::Token,
tokenizer::BorrowedToken,
};
impl Parser<'_> {
@ -74,18 +74,18 @@ impl Parser<'_> {
};
let using = if self.parse_keyword(Keyword::USING) {
self.expect_token(&Token::LParen)?;
self.expect_token(&BorrowedToken::LParen)?;
let expr = self.parse_expr()?;
self.expect_token(&Token::RParen)?;
self.expect_token(&BorrowedToken::RParen)?;
Some(expr)
} else {
None
};
let with_check = if self.parse_keywords(&[Keyword::WITH, Keyword::CHECK]) {
self.expect_token(&Token::LParen)?;
self.expect_token(&BorrowedToken::LParen)?;
let expr = self.parse_expr()?;
self.expect_token(&Token::RParen)?;
self.expect_token(&BorrowedToken::RParen)?;
Some(expr)
} else {
None
@ -216,7 +216,7 @@ impl Parser<'_> {
let add_mfa_method_otp =
if self.parse_keywords(&[Keyword::ADD, Keyword::MFA, Keyword::METHOD, Keyword::OTP]) {
let count = if self.parse_keyword(Keyword::COUNT) {
self.expect_token(&Token::Eq)?;
self.expect_token(&BorrowedToken::Eq)?;
Some(self.parse_value()?.into())
} else {
None
@ -336,7 +336,7 @@ impl Parser<'_> {
let member_name = self.parse_identifier()?;
AlterRoleOperation::DropMember { member_name }
} else if self.parse_keywords(&[Keyword::WITH, Keyword::NAME]) {
if self.consume_token(&Token::Eq) {
if self.consume_token(&BorrowedToken::Eq) {
let role_name = self.parse_identifier()?;
AlterRoleOperation::RenameRole { role_name }
} else {
@ -380,7 +380,7 @@ impl Parser<'_> {
in_database,
}
// { TO | = } { value | DEFAULT }
} else if self.consume_token(&Token::Eq) || self.parse_keyword(Keyword::TO) {
} else if self.consume_token(&BorrowedToken::Eq) || self.parse_keyword(Keyword::TO) {
if self.parse_keyword(Keyword::DEFAULT) {
AlterRoleOperation::Set {
config_name,

2108
src/parser/mod.rs
View file

File diff suppressed because it is too large Load diff

3
src/test_utils.rs
View file

@ -126,7 +126,8 @@ impl TestedDialects {
if let Some(options) = &self.options {
tokenizer = tokenizer.with_unescape(options.unescape);
}
let tokens = tokenizer.tokenize()?;
let tokens = tokenizer.tokenized_owned()?;
self.new_parser(dialect)
.with_tokens(tokens)
.parse_statements()

444
src/tokenizer.rs
View file

@ -51,11 +51,11 @@ use crate::dialect::{
use crate::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
use crate::{ast::DollarQuotedString, dialect::HiveDialect};
/// SQL Token enumeration
/// SQL Token enumeration with lifetime parameter for future zero-copy support
#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub enum Token {
pub enum BorrowedToken<'a> {
/// An end-of-file marker, not a real token
EOF,
/// A keyword (like SELECT) or an optionally quoted SQL identifier
@ -280,126 +280,284 @@ pub enum Token {
/// This is used to represent any custom binary operator that is not part of the SQL standard.
/// PostgreSQL allows defining custom binary operators using CREATE OPERATOR.
CustomBinaryOperator(String),
/// Marker to carry the lifetime parameter (never constructed)
_Phantom(Cow<'a, str>),
}
impl fmt::Display for Token {
/// Type alias for backward compatibility - Token without explicit lifetime uses 'static
pub type Token = BorrowedToken<'static>;
impl<'a> fmt::Display for BorrowedToken<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match self {
Token::EOF => f.write_str("EOF"),
Token::Word(ref w) => write!(f, "{w}"),
Token::Number(ref n, l) => write!(f, "{}{long}", n, long = if *l { "L" } else { "" }),
Token::Char(ref c) => write!(f, "{c}"),
Token::SingleQuotedString(ref s) => write!(f, "'{s}'"),
Token::TripleSingleQuotedString(ref s) => write!(f, "'''{s}'''"),
Token::DoubleQuotedString(ref s) => write!(f, "\"{s}\""),
Token::TripleDoubleQuotedString(ref s) => write!(f, "\"\"\"{s}\"\"\""),
Token::DollarQuotedString(ref s) => write!(f, "{s}"),
Token::NationalStringLiteral(ref s) => write!(f, "N'{s}'"),
Token::EscapedStringLiteral(ref s) => write!(f, "E'{s}'"),
Token::UnicodeStringLiteral(ref s) => write!(f, "U&'{s}'"),
Token::HexStringLiteral(ref s) => write!(f, "X'{s}'"),
Token::SingleQuotedByteStringLiteral(ref s) => write!(f, "B'{s}'"),
Token::TripleSingleQuotedByteStringLiteral(ref s) => write!(f, "B'''{s}'''"),
Token::DoubleQuotedByteStringLiteral(ref s) => write!(f, "B\"{s}\""),
Token::TripleDoubleQuotedByteStringLiteral(ref s) => write!(f, "B\"\"\"{s}\"\"\""),
Token::SingleQuotedRawStringLiteral(ref s) => write!(f, "R'{s}'"),
Token::DoubleQuotedRawStringLiteral(ref s) => write!(f, "R\"{s}\""),
Token::TripleSingleQuotedRawStringLiteral(ref s) => write!(f, "R'''{s}'''"),
Token::TripleDoubleQuotedRawStringLiteral(ref s) => write!(f, "R\"\"\"{s}\"\"\""),
Token::Comma => f.write_str(","),
Token::Whitespace(ws) => write!(f, "{ws}"),
Token::DoubleEq => f.write_str("=="),
Token::Spaceship => f.write_str("<=>"),
Token::Eq => f.write_str("="),
Token::Neq => f.write_str("<>"),
Token::Lt => f.write_str("<"),
Token::Gt => f.write_str(">"),
Token::LtEq => f.write_str("<="),
Token::GtEq => f.write_str(">="),
Token::Plus => f.write_str("+"),
Token::Minus => f.write_str("-"),
Token::Mul => f.write_str("*"),
Token::Div => f.write_str("/"),
Token::DuckIntDiv => f.write_str("//"),
Token::StringConcat => f.write_str("||"),
Token::Mod => f.write_str("%"),
Token::LParen => f.write_str("("),
Token::RParen => f.write_str(")"),
Token::Period => f.write_str("."),
Token::Colon => f.write_str(":"),
Token::DoubleColon => f.write_str("::"),
Token::Assignment => f.write_str(":="),
Token::SemiColon => f.write_str(";"),
Token::Backslash => f.write_str("\\"),
Token::LBracket => f.write_str("["),
Token::RBracket => f.write_str("]"),
Token::Ampersand => f.write_str("&"),
Token::Caret => f.write_str("^"),
Token::Pipe => f.write_str("|"),
Token::LBrace => f.write_str("{"),
Token::RBrace => f.write_str("}"),
Token::RArrow => f.write_str("=>"),
Token::Sharp => f.write_str("#"),
Token::DoubleSharp => f.write_str("##"),
Token::ExclamationMark => f.write_str("!"),
Token::DoubleExclamationMark => f.write_str("!!"),
Token::Tilde => f.write_str("~"),
Token::TildeAsterisk => f.write_str("~*"),
Token::ExclamationMarkTilde => f.write_str("!~"),
Token::ExclamationMarkTildeAsterisk => f.write_str("!~*"),
Token::DoubleTilde => f.write_str("~~"),
Token::DoubleTildeAsterisk => f.write_str("~~*"),
Token::ExclamationMarkDoubleTilde => f.write_str("!~~"),
Token::ExclamationMarkDoubleTildeAsterisk => f.write_str("!~~*"),
Token::AtSign => f.write_str("@"),
Token::CaretAt => f.write_str("^@"),
Token::ShiftLeft => f.write_str("<<"),
Token::ShiftRight => f.write_str(">>"),
Token::Overlap => f.write_str("&&"),
Token::PGSquareRoot => f.write_str("|/"),
Token::PGCubeRoot => f.write_str("||/"),
Token::AtDashAt => f.write_str("@-@"),
Token::QuestionMarkDash => f.write_str("?-"),
Token::AmpersandLeftAngleBracket => f.write_str("&<"),
Token::AmpersandRightAngleBracket => f.write_str("&>"),
Token::AmpersandLeftAngleBracketVerticalBar => f.write_str("&<|"),
Token::VerticalBarAmpersandRightAngleBracket => f.write_str("|&>"),
Token::VerticalBarRightAngleBracket => f.write_str("|>"),
Token::TwoWayArrow => f.write_str("<->"),
Token::LeftAngleBracketCaret => f.write_str("<^"),
Token::RightAngleBracketCaret => f.write_str(">^"),
Token::QuestionMarkSharp => f.write_str("?#"),
Token::QuestionMarkDashVerticalBar => f.write_str("?-|"),
Token::QuestionMarkDoubleVerticalBar => f.write_str("?||"),
Token::TildeEqual => f.write_str("~="),
Token::ShiftLeftVerticalBar => f.write_str("<<|"),
Token::VerticalBarShiftRight => f.write_str("|>>"),
Token::Placeholder(ref s) => write!(f, "{s}"),
Token::Arrow => write!(f, "->"),
Token::LongArrow => write!(f, "->>"),
Token::HashArrow => write!(f, "#>"),
Token::HashLongArrow => write!(f, "#>>"),
Token::AtArrow => write!(f, "@>"),
Token::ArrowAt => write!(f, "<@"),
Token::HashMinus => write!(f, "#-"),
Token::AtQuestion => write!(f, "@?"),
Token::AtAt => write!(f, "@@"),
Token::Question => write!(f, "?"),
Token::QuestionAnd => write!(f, "?&"),
Token::QuestionPipe => write!(f, "?|"),
Token::CustomBinaryOperator(s) => f.write_str(s),
BorrowedToken::EOF => f.write_str("EOF"),
BorrowedToken::Word(ref w) => write!(f, "{w}"),
BorrowedToken::Number(ref n, l) => {
write!(f, "{}{long}", n, long = if *l { "L" } else { "" })
}
BorrowedToken::Char(ref c) => write!(f, "{c}"),
BorrowedToken::SingleQuotedString(ref s) => write!(f, "'{s}'"),
BorrowedToken::TripleSingleQuotedString(ref s) => write!(f, "'''{s}'''"),
BorrowedToken::DoubleQuotedString(ref s) => write!(f, "\"{s}\""),
BorrowedToken::TripleDoubleQuotedString(ref s) => write!(f, "\"\"\"{s}\"\"\""),
BorrowedToken::DollarQuotedString(ref s) => write!(f, "{s}"),
BorrowedToken::NationalStringLiteral(ref s) => write!(f, "N'{s}'"),
BorrowedToken::EscapedStringLiteral(ref s) => write!(f, "E'{s}'"),
BorrowedToken::UnicodeStringLiteral(ref s) => write!(f, "U&'{s}'"),
BorrowedToken::HexStringLiteral(ref s) => write!(f, "X'{s}'"),
BorrowedToken::SingleQuotedByteStringLiteral(ref s) => write!(f, "B'{s}'"),
BorrowedToken::TripleSingleQuotedByteStringLiteral(ref s) => write!(f, "B'''{s}'''"),
BorrowedToken::DoubleQuotedByteStringLiteral(ref s) => write!(f, "B\"{s}\""),
BorrowedToken::TripleDoubleQuotedByteStringLiteral(ref s) => {
write!(f, "B\"\"\"{s}\"\"\"")
}
BorrowedToken::SingleQuotedRawStringLiteral(ref s) => write!(f, "R'{s}'"),
BorrowedToken::DoubleQuotedRawStringLiteral(ref s) => write!(f, "R\"{s}\""),
BorrowedToken::TripleSingleQuotedRawStringLiteral(ref s) => write!(f, "R'''{s}'''"),
BorrowedToken::TripleDoubleQuotedRawStringLiteral(ref s) => {
write!(f, "R\"\"\"{s}\"\"\"")
}
BorrowedToken::Comma => f.write_str(","),
BorrowedToken::Whitespace(ws) => write!(f, "{ws}"),
BorrowedToken::DoubleEq => f.write_str("=="),
BorrowedToken::Spaceship => f.write_str("<=>"),
BorrowedToken::Eq => f.write_str("="),
BorrowedToken::Neq => f.write_str("<>"),
BorrowedToken::Lt => f.write_str("<"),
BorrowedToken::Gt => f.write_str(">"),
BorrowedToken::LtEq => f.write_str("<="),
BorrowedToken::GtEq => f.write_str(">="),
BorrowedToken::Plus => f.write_str("+"),
BorrowedToken::Minus => f.write_str("-"),
BorrowedToken::Mul => f.write_str("*"),
BorrowedToken::Div => f.write_str("/"),
BorrowedToken::DuckIntDiv => f.write_str("//"),
BorrowedToken::StringConcat => f.write_str("||"),
BorrowedToken::Mod => f.write_str("%"),
BorrowedToken::LParen => f.write_str("("),
BorrowedToken::RParen => f.write_str(")"),
BorrowedToken::Period => f.write_str("."),
BorrowedToken::Colon => f.write_str(":"),
BorrowedToken::DoubleColon => f.write_str("::"),
BorrowedToken::Assignment => f.write_str(":="),
BorrowedToken::SemiColon => f.write_str(";"),
BorrowedToken::Backslash => f.write_str("\\"),
BorrowedToken::LBracket => f.write_str("["),
BorrowedToken::RBracket => f.write_str("]"),
BorrowedToken::Ampersand => f.write_str("&"),
BorrowedToken::Caret => f.write_str("^"),
BorrowedToken::Pipe => f.write_str("|"),
BorrowedToken::LBrace => f.write_str("{"),
BorrowedToken::RBrace => f.write_str("}"),
BorrowedToken::RArrow => f.write_str("=>"),
BorrowedToken::Sharp => f.write_str("#"),
BorrowedToken::DoubleSharp => f.write_str("##"),
BorrowedToken::ExclamationMark => f.write_str("!"),
BorrowedToken::DoubleExclamationMark => f.write_str("!!"),
BorrowedToken::Tilde => f.write_str("~"),
BorrowedToken::TildeAsterisk => f.write_str("~*"),
BorrowedToken::ExclamationMarkTilde => f.write_str("!~"),
BorrowedToken::ExclamationMarkTildeAsterisk => f.write_str("!~*"),
BorrowedToken::DoubleTilde => f.write_str("~~"),
BorrowedToken::DoubleTildeAsterisk => f.write_str("~~*"),
BorrowedToken::ExclamationMarkDoubleTilde => f.write_str("!~~"),
BorrowedToken::ExclamationMarkDoubleTildeAsterisk => f.write_str("!~~*"),
BorrowedToken::AtSign => f.write_str("@"),
BorrowedToken::CaretAt => f.write_str("^@"),
BorrowedToken::ShiftLeft => f.write_str("<<"),
BorrowedToken::ShiftRight => f.write_str(">>"),
BorrowedToken::Overlap => f.write_str("&&"),
BorrowedToken::PGSquareRoot => f.write_str("|/"),
BorrowedToken::PGCubeRoot => f.write_str("||/"),
BorrowedToken::AtDashAt => f.write_str("@-@"),
BorrowedToken::QuestionMarkDash => f.write_str("?-"),
BorrowedToken::AmpersandLeftAngleBracket => f.write_str("&<"),
BorrowedToken::AmpersandRightAngleBracket => f.write_str("&>"),
BorrowedToken::AmpersandLeftAngleBracketVerticalBar => f.write_str("&<|"),
BorrowedToken::VerticalBarAmpersandRightAngleBracket => f.write_str("|&>"),
BorrowedToken::VerticalBarRightAngleBracket => f.write_str("|>"),
BorrowedToken::TwoWayArrow => f.write_str("<->"),
BorrowedToken::LeftAngleBracketCaret => f.write_str("<^"),
BorrowedToken::RightAngleBracketCaret => f.write_str(">^"),
BorrowedToken::QuestionMarkSharp => f.write_str("?#"),
BorrowedToken::QuestionMarkDashVerticalBar => f.write_str("?-|"),
BorrowedToken::QuestionMarkDoubleVerticalBar => f.write_str("?||"),
BorrowedToken::TildeEqual => f.write_str("~="),
BorrowedToken::ShiftLeftVerticalBar => f.write_str("<<|"),
BorrowedToken::VerticalBarShiftRight => f.write_str("|>>"),
BorrowedToken::Placeholder(ref s) => write!(f, "{s}"),
BorrowedToken::Arrow => write!(f, "->"),
BorrowedToken::LongArrow => write!(f, "->>"),
BorrowedToken::HashArrow => write!(f, "#>"),
BorrowedToken::HashLongArrow => write!(f, "#>>"),
BorrowedToken::AtArrow => write!(f, "@>"),
BorrowedToken::ArrowAt => write!(f, "<@"),
BorrowedToken::HashMinus => write!(f, "#-"),
BorrowedToken::AtQuestion => write!(f, "@?"),
BorrowedToken::AtAt => write!(f, "@@"),
BorrowedToken::Question => write!(f, "?"),
BorrowedToken::QuestionAnd => write!(f, "?&"),
BorrowedToken::QuestionPipe => write!(f, "?|"),
BorrowedToken::CustomBinaryOperator(s) => f.write_str(s),
BorrowedToken::_Phantom(_) => unreachable!("_Phantom should never be constructed"),
}
}
}
impl Token {
impl<'a> BorrowedToken<'a> {
/// Converts a borrowed token to a static token by taking ownership and moving the data
pub fn to_static(self) -> Token {
match self {
BorrowedToken::EOF => BorrowedToken::EOF,
BorrowedToken::Word(w) => BorrowedToken::Word(w),
BorrowedToken::Number(n, l) => BorrowedToken::Number(n, l),
BorrowedToken::Char(c) => BorrowedToken::Char(c),
BorrowedToken::SingleQuotedString(s) => BorrowedToken::SingleQuotedString(s),
BorrowedToken::DoubleQuotedString(s) => BorrowedToken::DoubleQuotedString(s),
BorrowedToken::TripleSingleQuotedString(s) => {
BorrowedToken::TripleSingleQuotedString(s)
}
BorrowedToken::TripleDoubleQuotedString(s) => {
BorrowedToken::TripleDoubleQuotedString(s)
}
BorrowedToken::DollarQuotedString(s) => BorrowedToken::DollarQuotedString(s),
BorrowedToken::SingleQuotedByteStringLiteral(s) => {
BorrowedToken::SingleQuotedByteStringLiteral(s)
}
BorrowedToken::DoubleQuotedByteStringLiteral(s) => {
BorrowedToken::DoubleQuotedByteStringLiteral(s)
}
BorrowedToken::TripleSingleQuotedByteStringLiteral(s) => {
BorrowedToken::TripleSingleQuotedByteStringLiteral(s)
}
BorrowedToken::TripleDoubleQuotedByteStringLiteral(s) => {
BorrowedToken::TripleDoubleQuotedByteStringLiteral(s)
}
BorrowedToken::SingleQuotedRawStringLiteral(s) => {
BorrowedToken::SingleQuotedRawStringLiteral(s)
}
BorrowedToken::DoubleQuotedRawStringLiteral(s) => {
BorrowedToken::DoubleQuotedRawStringLiteral(s)
}
BorrowedToken::TripleSingleQuotedRawStringLiteral(s) => {
BorrowedToken::TripleSingleQuotedRawStringLiteral(s)
}
BorrowedToken::TripleDoubleQuotedRawStringLiteral(s) => {
BorrowedToken::TripleDoubleQuotedRawStringLiteral(s)
}
BorrowedToken::NationalStringLiteral(s) => BorrowedToken::NationalStringLiteral(s),
BorrowedToken::EscapedStringLiteral(s) => BorrowedToken::EscapedStringLiteral(s),
BorrowedToken::UnicodeStringLiteral(s) => BorrowedToken::UnicodeStringLiteral(s),
BorrowedToken::HexStringLiteral(s) => BorrowedToken::HexStringLiteral(s),
BorrowedToken::Comma => BorrowedToken::Comma,
BorrowedToken::Whitespace(ws) => BorrowedToken::Whitespace(ws),
BorrowedToken::DoubleEq => BorrowedToken::DoubleEq,
BorrowedToken::Eq => BorrowedToken::Eq,
BorrowedToken::Neq => BorrowedToken::Neq,
BorrowedToken::Lt => BorrowedToken::Lt,
BorrowedToken::Gt => BorrowedToken::Gt,
BorrowedToken::LtEq => BorrowedToken::LtEq,
BorrowedToken::GtEq => BorrowedToken::GtEq,
BorrowedToken::Spaceship => BorrowedToken::Spaceship,
BorrowedToken::Plus => BorrowedToken::Plus,
BorrowedToken::Minus => BorrowedToken::Minus,
BorrowedToken::Mul => BorrowedToken::Mul,
BorrowedToken::Div => BorrowedToken::Div,
BorrowedToken::DuckIntDiv => BorrowedToken::DuckIntDiv,
BorrowedToken::Mod => BorrowedToken::Mod,
BorrowedToken::StringConcat => BorrowedToken::StringConcat,
BorrowedToken::LParen => BorrowedToken::LParen,
BorrowedToken::RParen => BorrowedToken::RParen,
BorrowedToken::Period => BorrowedToken::Period,
BorrowedToken::Colon => BorrowedToken::Colon,
BorrowedToken::DoubleColon => BorrowedToken::DoubleColon,
BorrowedToken::Assignment => BorrowedToken::Assignment,
BorrowedToken::SemiColon => BorrowedToken::SemiColon,
BorrowedToken::Backslash => BorrowedToken::Backslash,
BorrowedToken::LBracket => BorrowedToken::LBracket,
BorrowedToken::RBracket => BorrowedToken::RBracket,
BorrowedToken::Ampersand => BorrowedToken::Ampersand,
BorrowedToken::Pipe => BorrowedToken::Pipe,
BorrowedToken::Caret => BorrowedToken::Caret,
BorrowedToken::LBrace => BorrowedToken::LBrace,
BorrowedToken::RBrace => BorrowedToken::RBrace,
BorrowedToken::RArrow => BorrowedToken::RArrow,
BorrowedToken::Sharp => BorrowedToken::Sharp,
BorrowedToken::DoubleSharp => BorrowedToken::DoubleSharp,
BorrowedToken::Tilde => BorrowedToken::Tilde,
BorrowedToken::TildeAsterisk => BorrowedToken::TildeAsterisk,
BorrowedToken::ExclamationMarkTilde => BorrowedToken::ExclamationMarkTilde,
BorrowedToken::ExclamationMarkTildeAsterisk => {
BorrowedToken::ExclamationMarkTildeAsterisk
}
BorrowedToken::DoubleTilde => BorrowedToken::DoubleTilde,
BorrowedToken::DoubleTildeAsterisk => BorrowedToken::DoubleTildeAsterisk,
BorrowedToken::ExclamationMarkDoubleTilde => BorrowedToken::ExclamationMarkDoubleTilde,
BorrowedToken::ExclamationMarkDoubleTildeAsterisk => {
BorrowedToken::ExclamationMarkDoubleTildeAsterisk
}
BorrowedToken::ShiftLeft => BorrowedToken::ShiftLeft,
BorrowedToken::ShiftRight => BorrowedToken::ShiftRight,
BorrowedToken::Overlap => BorrowedToken::Overlap,
BorrowedToken::ExclamationMark => BorrowedToken::ExclamationMark,
BorrowedToken::DoubleExclamationMark => BorrowedToken::DoubleExclamationMark,
BorrowedToken::AtSign => BorrowedToken::AtSign,
BorrowedToken::CaretAt => BorrowedToken::CaretAt,
BorrowedToken::PGSquareRoot => BorrowedToken::PGSquareRoot,
BorrowedToken::PGCubeRoot => BorrowedToken::PGCubeRoot,
BorrowedToken::Placeholder(s) => BorrowedToken::Placeholder(s),
BorrowedToken::Arrow => BorrowedToken::Arrow,
BorrowedToken::LongArrow => BorrowedToken::LongArrow,
BorrowedToken::HashArrow => BorrowedToken::HashArrow,
BorrowedToken::AtDashAt => BorrowedToken::AtDashAt,
BorrowedToken::QuestionMarkDash => BorrowedToken::QuestionMarkDash,
BorrowedToken::AmpersandLeftAngleBracket => BorrowedToken::AmpersandLeftAngleBracket,
BorrowedToken::AmpersandRightAngleBracket => BorrowedToken::AmpersandRightAngleBracket,
BorrowedToken::AmpersandLeftAngleBracketVerticalBar => {
BorrowedToken::AmpersandLeftAngleBracketVerticalBar
}
BorrowedToken::VerticalBarAmpersandRightAngleBracket => {
BorrowedToken::VerticalBarAmpersandRightAngleBracket
}
BorrowedToken::TwoWayArrow => BorrowedToken::TwoWayArrow,
BorrowedToken::LeftAngleBracketCaret => BorrowedToken::LeftAngleBracketCaret,
BorrowedToken::RightAngleBracketCaret => BorrowedToken::RightAngleBracketCaret,
BorrowedToken::QuestionMarkSharp => BorrowedToken::QuestionMarkSharp,
BorrowedToken::QuestionMarkDashVerticalBar => {
BorrowedToken::QuestionMarkDashVerticalBar
}
BorrowedToken::QuestionMarkDoubleVerticalBar => {
BorrowedToken::QuestionMarkDoubleVerticalBar
}
BorrowedToken::TildeEqual => BorrowedToken::TildeEqual,
BorrowedToken::ShiftLeftVerticalBar => BorrowedToken::ShiftLeftVerticalBar,
BorrowedToken::VerticalBarShiftRight => BorrowedToken::VerticalBarShiftRight,
BorrowedToken::VerticalBarRightAngleBracket => {
BorrowedToken::VerticalBarRightAngleBracket
}
BorrowedToken::HashLongArrow => BorrowedToken::HashLongArrow,
BorrowedToken::AtArrow => BorrowedToken::AtArrow,
BorrowedToken::ArrowAt => BorrowedToken::ArrowAt,
BorrowedToken::HashMinus => BorrowedToken::HashMinus,
BorrowedToken::AtQuestion => BorrowedToken::AtQuestion,
BorrowedToken::AtAt => BorrowedToken::AtAt,
BorrowedToken::Question => BorrowedToken::Question,
BorrowedToken::QuestionAnd => BorrowedToken::QuestionAnd,
BorrowedToken::QuestionPipe => BorrowedToken::QuestionPipe,
BorrowedToken::CustomBinaryOperator(s) => BorrowedToken::CustomBinaryOperator(s),
BorrowedToken::_Phantom(_) => unreachable!("_Phantom should never be constructed"),
}
}
}
impl BorrowedToken<'static> {
pub fn make_keyword(keyword: &str) -> Self {
Token::make_word(keyword, None)
BorrowedToken::make_word(keyword, None)
}
pub fn make_word(word: &str, quote_style: Option<char>) -> Self {
let word_uppercase = word.to_uppercase();
Token::Word(Word {
BorrowedToken::Word(Word {
value: word.to_string(),
quote_style,
keyword: if quote_style.is_none() {
@ -656,7 +814,7 @@ impl Span {
/// Backwards compatibility struct for [`TokenWithSpan`]
#[deprecated(since = "0.53.0", note = "please use `TokenWithSpan` instead")]
pub type TokenWithLocation = TokenWithSpan;
pub type TokenWithLocation<'a> = TokenWithSpan<'a>;
/// A [Token] with [Span] attached to it
///
@ -683,46 +841,58 @@ pub type TokenWithLocation = TokenWithSpan;
#[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct TokenWithSpan {
pub token: Token,
pub struct TokenWithSpan<'a> {
pub token: BorrowedToken<'a>,
pub span: Span,
}
impl TokenWithSpan {
/// Create a new [`TokenWithSpan`] from a [`Token`] and a [`Span`]
pub fn new(token: Token, span: Span) -> Self {
impl<'a> TokenWithSpan<'a> {
/// Create a new [`TokenWithSpan`] from a [`BorrowedToken`] and a [`Span`]
pub fn new(token: BorrowedToken<'a>, span: Span) -> Self {
Self { token, span }
}
/// Wrap a token with an empty span
pub fn wrap(token: Token) -> Self {
pub fn wrap(token: BorrowedToken<'a>) -> Self {
Self::new(token, Span::empty())
}
/// Wrap a token with a location from `start` to `end`
pub fn at(token: Token, start: Location, end: Location) -> Self {
pub fn at(token: BorrowedToken<'a>, start: Location, end: Location) -> Self {
Self::new(token, Span::new(start, end))
}
/// Return an EOF token with no location
pub fn new_eof() -> Self {
Self::wrap(Token::EOF)
Self::wrap(BorrowedToken::EOF)
}
/// Convert to a `'static` lifetime by cloning the underlying data.
///
/// This is used when tokens need to be stored in AST nodes that must be owned.
/// Currently all data is already owned (String), so this is just a clone.
/// When Cow is introduced, this will convert Cow::Borrowed → Cow::Owned.
pub fn to_static(self) -> TokenWithSpan<'static> {
TokenWithSpan {
token: self.token.to_static(),
span: self.span,
}
}
}
impl PartialEq<Token> for TokenWithSpan {
fn eq(&self, other: &Token) -> bool {
impl<'a> PartialEq<BorrowedToken<'a>> for TokenWithSpan<'a> {
fn eq(&self, other: &BorrowedToken<'a>) -> bool {
&self.token == other
}
}
impl PartialEq<TokenWithSpan> for Token {
fn eq(&self, other: &TokenWithSpan) -> bool {
impl<'a> PartialEq<TokenWithSpan<'a>> for BorrowedToken<'a> {
fn eq(&self, other: &TokenWithSpan<'a>) -> bool {
self == &other.token
}
}
impl fmt::Display for TokenWithSpan {
impl<'a> fmt::Display for TokenWithSpan<'a> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.token.fmt(f)
}
@ -892,23 +1062,35 @@ impl<'a> Tokenizer<'a> {
}
/// Tokenize the statement and produce a vector of tokens
pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {
pub fn tokenize(&mut self) -> Result<Vec<BorrowedToken<'a>>, TokenizerError> {
let twl = self.tokenize_with_location()?;
Ok(twl.into_iter().map(|t| t.token).collect())
}
pub fn tokenized_owned(&mut self) -> Result<Vec<Token>, TokenizerError> {
let tokens = self.tokenize()?;
Ok(tokens.into_iter().map(|t| t.to_static()).collect())
}
/// Tokenize the statement and produce a vector of tokens with location information
pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithSpan>, TokenizerError> {
let mut tokens: Vec<TokenWithSpan> = vec![];
pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithSpan<'a>>, TokenizerError> {
let mut tokens: Vec<TokenWithSpan<'a>> = vec![];
self.tokenize_with_location_into_buf(&mut tokens)
.map(|_| tokens)
}
pub fn tokenized_with_location_owned(
&mut self,
) -> Result<Vec<TokenWithSpan<'static>>, TokenizerError> {
let tokens = self.tokenize_with_location()?;
Ok(tokens.into_iter().map(|t| t.to_static()).collect())
}
/// Tokenize the statement and append tokens with location information into the provided buffer.
/// If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error.
pub fn tokenize_with_location_into_buf(
&mut self,
buf: &mut Vec<TokenWithSpan>,
buf: &mut Vec<TokenWithSpan<'a>>,
) -> Result<(), TokenizerError> {
let mut state = State {
peekable: self.query.chars().peekable(),
@ -961,7 +1143,7 @@ impl<'a> Tokenizer<'a> {
fn next_token(
&self,
chars: &mut State<'a>,
prev_token: Option<&Token>,
prev_token: Option<&BorrowedToken<'a>>,
) -> Result<Option<Token>, TokenizerError> {
match chars.peek() {
Some(&ch) => match ch {
@ -1219,7 +1401,7 @@ impl<'a> Tokenizer<'a> {
// if the prev token is not a word, then this is not a valid sql
// word or number.
if ch == '.' && chars.peekable.clone().nth(1) == Some('_') {
if let Some(Token::Word(_)) = prev_token {
if let Some(&BorrowedToken::Word(_)) = prev_token {
chars.next();
return Ok(Some(Token::Period));
}
@ -1263,7 +1445,7 @@ impl<'a> Tokenizer<'a> {
// we should yield the dot as a dedicated token so compound identifiers
// starting with digits can be parsed correctly.
if s == "." && self.dialect.supports_numeric_prefix() {
if let Some(Token::Word(_)) = prev_token {
if let Some(&BorrowedToken::Word(_)) = prev_token {
return Ok(Some(Token::Period));
}
}
@ -1322,7 +1504,7 @@ impl<'a> Tokenizer<'a> {
s += word.as_str();
return Ok(Some(Token::make_word(s.as_str(), None)));
}
} else if prev_token == Some(&Token::Period) {
} else if matches!(prev_token, Some(&BorrowedToken::Period)) {
// If the previous token was a period, thus not belonging to a number,
// the value we have is part of an identifier.
return Ok(Some(Token::make_word(s.as_str(), None)));

2
tests/sqlparser_postgres.rs
View file

@ -2045,7 +2045,7 @@ fn parse_pg_returning() {
fn test_operator(operator: &str, dialect: &TestedDialects, expected: BinaryOperator) {
let operator_tokens =
sqlparser::tokenizer::Tokenizer::new(&PostgreSqlDialect {}, &format!("a{operator}b"))
.tokenize()
.tokenized_owned()
.unwrap();
assert_eq!(
operator_tokens.len(),