mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-25 08:24:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Rename TokenWithLocation to TokenWithSpan, in backwards compatible way (#1562)

Browse source
This commit is contained in:
Andrew Lamb 2024-11-30 07:55:21 -05:00 • committed by GitHub
parent 92c6e7f79b
commit a134910a36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 67 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/ast/helpers/attached_token.rs
View file

@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
use core::fmt::{self, Debug, Formatter}; use core::fmt::{self, Debug, Formatter};
use core::hash::{Hash, Hasher}; use core::hash::{Hash, Hasher};
use crate::tokenizer::{Token, TokenWithLocation}; use crate::tokenizer::{Token, TokenWithSpan};
#[cfg(feature = "serde")] #[cfg(feature = "serde")]
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut};
#[derive(Clone)] #[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct AttachedToken(pub TokenWithLocation); pub struct AttachedToken(pub TokenWithSpan);
impl AttachedToken { impl AttachedToken {
pub fn empty() -> Self { pub fn empty() -> Self {
AttachedToken(TokenWithLocation::wrap(Token::EOF)) AttachedToken(TokenWithSpan::wrap(Token::EOF))
} }
} }
@ -75,8 +75,8 @@ impl Hash for AttachedToken {
} }
} }
impl From<TokenWithLocation> for AttachedToken { impl From<TokenWithSpan> for AttachedToken {
fn from(value: TokenWithLocation) -> Self { fn from(value: TokenWithSpan) -> Self {
AttachedToken(value) AttachedToken(value)
} }
} }

4
src/ast/query.rs
View file

@ -27,7 +27,7 @@ use sqlparser_derive::{Visit, VisitMut};
use crate::{ use crate::{
ast::*, ast::*,
tokenizer::{Token, TokenWithLocation}, tokenizer::{Token, TokenWithSpan},
}; };
/// The most complete variant of a `SELECT` query expression, optionally /// The most complete variant of a `SELECT` query expression, optionally
@ -643,7 +643,7 @@ pub struct WildcardAdditionalOptions {
impl Default for WildcardAdditionalOptions { impl Default for WildcardAdditionalOptions {
fn default() -> Self { fn default() -> Self {
Self { Self {
wildcard_token: TokenWithLocation::wrap(Token::Mul).into(), wildcard_token: TokenWithSpan::wrap(Token::Mul).into(),
opt_ilike: None, opt_ilike: None,
opt_exclude: None, opt_exclude: None,
opt_except: None, opt_except: None,

66
src/parser/mod.rs
View file

@ -265,7 +265,7 @@ enum ParserState {
} }
pub struct Parser<'a> { pub struct Parser<'a> {
tokens: Vec<TokenWithLocation>, tokens: Vec<TokenWithSpan>,
/// The index of the first unprocessed token in [`Parser::tokens`]. /// The index of the first unprocessed token in [`Parser::tokens`].
index: usize, index: usize,
/// The current state of the parser. /// The current state of the parser.
@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
} }
/// Reset this parser to parse the specified token stream /// Reset this parser to parse the specified token stream
pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithLocation>) -> Self { pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithSpan>) -> Self {
self.tokens = tokens; self.tokens = tokens;
self.index = 0; self.index = 0;
self self
@ -368,9 +368,9 @@ impl<'a> Parser<'a> {
/// Reset this parser state to parse the specified tokens /// Reset this parser state to parse the specified tokens
pub fn with_tokens(self, tokens: Vec<Token>) -> Self { pub fn with_tokens(self, tokens: Vec<Token>) -> Self {
// Put in dummy locations // Put in dummy locations
let tokens_with_locations: Vec<TokenWithLocation> = tokens let tokens_with_locations: Vec<TokenWithSpan> = tokens
.into_iter() .into_iter()
.map(|token| TokenWithLocation { .map(|token| TokenWithSpan {
token, token,
span: Span::empty(), span: Span::empty(),
}) })
@ -1147,7 +1147,7 @@ impl<'a> Parser<'a> {
match self.peek_token().token { match self.peek_token().token {
Token::LParen | Token::Period => { Token::LParen | Token::Period => {
let mut id_parts: Vec<Ident> = vec![w.to_ident(w_span)]; let mut id_parts: Vec<Ident> = vec![w.to_ident(w_span)];
let mut ending_wildcard: Option<TokenWithLocation> = None; let mut ending_wildcard: Option<TokenWithSpan> = None;
while self.consume_token(&Token::Period) { while self.consume_token(&Token::Period) {
let next_token = self.next_token(); let next_token = self.next_token();
match next_token.token { match next_token.token {
@ -3273,7 +3273,7 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed /// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file) /// (or None if reached end-of-file)
pub fn peek_token(&self) -> TokenWithLocation { pub fn peek_token(&self) -> TokenWithSpan {
self.peek_nth_token(0) self.peek_nth_token(0)
} }
@ -3308,19 +3308,19 @@ impl<'a> Parser<'a> {
/// yet been processed. /// yet been processed.
/// ///
/// See [`Self::peek_token`] for an example. /// See [`Self::peek_token`] for an example.
pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithLocation; N] { pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithSpan; N] {
let mut index = self.index; let mut index = self.index;
core::array::from_fn(|_| loop { core::array::from_fn(|_| loop {
let token = self.tokens.get(index); let token = self.tokens.get(index);
index += 1; index += 1;
if let Some(TokenWithLocation { if let Some(TokenWithSpan {
token: Token::Whitespace(_), token: Token::Whitespace(_),
span: _, span: _,
}) = token }) = token
{ {
continue; continue;
} }
break token.cloned().unwrap_or(TokenWithLocation { break token.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF, token: Token::EOF,
span: Span::empty(), span: Span::empty(),
}); });
@ -3328,18 +3328,18 @@ impl<'a> Parser<'a> {
} }
/// Return nth non-whitespace token that has not yet been processed /// Return nth non-whitespace token that has not yet been processed
pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation { pub fn peek_nth_token(&self, mut n: usize) -> TokenWithSpan {
let mut index = self.index; let mut index = self.index;
loop { loop {
index += 1; index += 1;
match self.tokens.get(index - 1) { match self.tokens.get(index - 1) {
Some(TokenWithLocation { Some(TokenWithSpan {
token: Token::Whitespace(_), token: Token::Whitespace(_),
span: _, span: _,
}) => continue, }) => continue,
non_whitespace => { non_whitespace => {
if n == 0 { if n == 0 {
return non_whitespace.cloned().unwrap_or(TokenWithLocation { return non_whitespace.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF, token: Token::EOF,
span: Span::empty(), span: Span::empty(),
}); });
@ -3352,16 +3352,16 @@ impl<'a> Parser<'a> {
/// Return the first token, possibly whitespace, that has not yet been processed /// Return the first token, possibly whitespace, that has not yet been processed
/// (or None if reached end-of-file). /// (or None if reached end-of-file).
pub fn peek_token_no_skip(&self) -> TokenWithLocation { pub fn peek_token_no_skip(&self) -> TokenWithSpan {
self.peek_nth_token_no_skip(0) self.peek_nth_token_no_skip(0)
} }
/// Return nth token, possibly whitespace, that has not yet been processed. /// Return nth token, possibly whitespace, that has not yet been processed.
pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithLocation { pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithSpan {
self.tokens self.tokens
.get(self.index + n) .get(self.index + n)
.cloned() .cloned()
.unwrap_or(TokenWithLocation { .unwrap_or(TokenWithSpan {
token: Token::EOF, token: Token::EOF,
span: Span::empty(), span: Span::empty(),
}) })
@ -3378,25 +3378,25 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed /// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file) and mark it as processed. OK to call /// (or None if reached end-of-file) and mark it as processed. OK to call
/// repeatedly after reaching EOF. /// repeatedly after reaching EOF.
pub fn next_token(&mut self) -> TokenWithLocation { pub fn next_token(&mut self) -> TokenWithSpan {
loop { loop {
self.index += 1; self.index += 1;
match self.tokens.get(self.index - 1) { match self.tokens.get(self.index - 1) {
Some(TokenWithLocation { Some(TokenWithSpan {
token: Token::Whitespace(_), token: Token::Whitespace(_),
span: _, span: _,
}) => continue, }) => continue,
token => { token => {
return token return token
.cloned() .cloned()
.unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF)) .unwrap_or_else(|| TokenWithSpan::wrap(Token::EOF))
} }
} }
} }
} }
/// Return the first unprocessed token, possibly whitespace. /// Return the first unprocessed token, possibly whitespace.
pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> { pub fn next_token_no_skip(&mut self) -> Option<&TokenWithSpan> {
self.index += 1; self.index += 1;
self.tokens.get(self.index - 1) self.tokens.get(self.index - 1)
} }
@ -3408,7 +3408,7 @@ impl<'a> Parser<'a> {
loop { loop {
assert!(self.index > 0); assert!(self.index > 0);
self.index -= 1; self.index -= 1;
if let Some(TokenWithLocation { if let Some(TokenWithSpan {
token: Token::Whitespace(_), token: Token::Whitespace(_),
span: _, span: _,
}) = self.tokens.get(self.index) }) = self.tokens.get(self.index)
@ -3420,7 +3420,7 @@ impl<'a> Parser<'a> {
} }
/// Report `found` was encountered instead of `expected` /// Report `found` was encountered instead of `expected`
pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) -> Result<T, ParserError> { pub fn expected<T>(&self, expected: &str, found: TokenWithSpan) -> Result<T, ParserError> {
parser_err!( parser_err!(
format!("Expected: {expected}, found: {found}"), format!("Expected: {expected}, found: {found}"),
found.span.start found.span.start
@ -3435,7 +3435,7 @@ impl<'a> Parser<'a> {
} }
#[must_use] #[must_use]
pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithLocation> { pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithSpan> {
match self.peek_token().token { match self.peek_token().token {
Token::Word(w) if expected == w.keyword => Some(self.next_token()), Token::Word(w) if expected == w.keyword => Some(self.next_token()),
_ => None, _ => None,
@ -3524,7 +3524,7 @@ impl<'a> Parser<'a> {
/// If the current token is the `expected` keyword, consume the token. /// If the current token is the `expected` keyword, consume the token.
/// Otherwise, return an error. /// Otherwise, return an error.
pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithLocation, ParserError> { pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithSpan, ParserError> {
if let Some(token) = self.parse_keyword_token(expected) { if let Some(token) = self.parse_keyword_token(expected) {
Ok(token) Ok(token)
} else { } else {
@ -3568,7 +3568,7 @@ impl<'a> Parser<'a> {
} }
/// Bail out if the current token is not an expected keyword, or consume it if it is /// Bail out if the current token is not an expected keyword, or consume it if it is
pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithLocation, ParserError> { pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithSpan, ParserError> {
if self.peek_token() == *expected { if self.peek_token() == *expected {
Ok(self.next_token()) Ok(self.next_token())
} else { } else {
@ -4107,7 +4107,7 @@ impl<'a> Parser<'a> {
Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))), Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))),
_ => self.expected( _ => self.expected(
"JAR, FILE or ARCHIVE, got {:?}", "JAR, FILE or ARCHIVE, got {:?}",
TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())), TokenWithSpan::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
), ),
} }
} }
@ -6832,7 +6832,7 @@ impl<'a> Parser<'a> {
if let Some(name) = name { if let Some(name) = name {
return self.expected( return self.expected(
"FULLTEXT or SPATIAL option without constraint name", "FULLTEXT or SPATIAL option without constraint name",
TokenWithLocation { TokenWithSpan {
token: Token::make_keyword(&name.to_string()), token: Token::make_keyword(&name.to_string()),
span: next_token.span, span: next_token.span,
}, },
@ -7808,7 +7808,7 @@ impl<'a> Parser<'a> {
Some('\'') => Ok(Value::SingleQuotedString(w.value)), Some('\'') => Ok(Value::SingleQuotedString(w.value)),
_ => self.expected( _ => self.expected(
"A value?", "A value?",
TokenWithLocation { TokenWithSpan {
token: Token::Word(w), token: Token::Word(w),
span, span,
}, },
@ -7816,7 +7816,7 @@ impl<'a> Parser<'a> {
}, },
_ => self.expected( _ => self.expected(
"a concrete value", "a concrete value",
TokenWithLocation { TokenWithSpan {
token: Token::Word(w), token: Token::Word(w),
span, span,
}, },
@ -7878,7 +7878,7 @@ impl<'a> Parser<'a> {
} }
unexpected => self.expected( unexpected => self.expected(
"a value", "a value",
TokenWithLocation { TokenWithSpan {
token: unexpected, token: unexpected,
span, span,
}, },
@ -7927,7 +7927,7 @@ impl<'a> Parser<'a> {
Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())), Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
unexpected => self.expected( unexpected => self.expected(
"a string value", "a string value",
TokenWithLocation { TokenWithSpan {
token: unexpected, token: unexpected,
span, span,
}, },
@ -8618,7 +8618,7 @@ impl<'a> Parser<'a> {
let token = self let token = self
.next_token_no_skip() .next_token_no_skip()
.cloned() .cloned()
.unwrap_or(TokenWithLocation::wrap(Token::EOF)); .unwrap_or(TokenWithSpan::wrap(Token::EOF));
requires_whitespace = match token.token { requires_whitespace = match token.token {
Token::Word(next_word) if next_word.quote_style.is_none() => { Token::Word(next_word) if next_word.quote_style.is_none() => {
ident.value.push_str(&next_word.value); ident.value.push_str(&next_word.value);
@ -11683,7 +11683,7 @@ impl<'a> Parser<'a> {
/// If it is not possible to parse it, will return an option. /// If it is not possible to parse it, will return an option.
pub fn parse_wildcard_additional_options( pub fn parse_wildcard_additional_options(
&mut self, &mut self,
wildcard_token: TokenWithLocation, wildcard_token: TokenWithSpan,
) -> Result<WildcardAdditionalOptions, ParserError> { ) -> Result<WildcardAdditionalOptions, ParserError> {
let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) { let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) {
self.parse_optional_select_item_ilike()? self.parse_optional_select_item_ilike()?
@ -12708,7 +12708,7 @@ impl<'a> Parser<'a> {
} }
/// Consume the parser and return its underlying token buffer /// Consume the parser and return its underlying token buffer
pub fn into_tokens(self) -> Vec<TokenWithLocation> { pub fn into_tokens(self) -> Vec<TokenWithSpan> {
self.tokens self.tokens
} }

50
src/tokenizer.rs
View file

@ -521,42 +521,46 @@ impl Span {
} }
} }
/// Backwards compatibility struct for [`TokenWithSpan`]
#[deprecated(since = "0.53.0", note = "please use `TokenWithSpan` instead")]
pub type TokenWithLocation = TokenWithSpan;
/// A [Token] with [Location] attached to it /// A [Token] with [Location] attached to it
#[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)] #[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct TokenWithLocation { pub struct TokenWithSpan {
pub token: Token, pub token: Token,
pub span: Span, pub span: Span,
} }
impl TokenWithLocation { impl TokenWithSpan {
pub fn new(token: Token, span: Span) -> TokenWithLocation { pub fn new(token: Token, span: Span) -> TokenWithSpan {
TokenWithLocation { token, span } TokenWithSpan { token, span }
} }
pub fn wrap(token: Token) -> TokenWithLocation { pub fn wrap(token: Token) -> TokenWithSpan {
TokenWithLocation::new(token, Span::empty()) TokenWithSpan::new(token, Span::empty())
} }
pub fn at(token: Token, start: Location, end: Location) -> TokenWithLocation { pub fn at(token: Token, start: Location, end: Location) -> TokenWithSpan {
TokenWithLocation::new(token, Span::new(start, end)) TokenWithSpan::new(token, Span::new(start, end))
} }
} }
impl PartialEq<Token> for TokenWithLocation { impl PartialEq<Token> for TokenWithSpan {
fn eq(&self, other: &Token) -> bool { fn eq(&self, other: &Token) -> bool {
&self.token == other &self.token == other
} }
} }
impl PartialEq<TokenWithLocation> for Token { impl PartialEq<TokenWithSpan> for Token {
fn eq(&self, other: &TokenWithLocation) -> bool { fn eq(&self, other: &TokenWithSpan) -> bool {
self == &other.token self == &other.token
} }
} }
impl fmt::Display for TokenWithLocation { impl fmt::Display for TokenWithSpan {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.token.fmt(f) self.token.fmt(f)
} }
@ -716,8 +720,8 @@ impl<'a> Tokenizer<'a> {
} }
/// Tokenize the statement and produce a vector of tokens with location information /// Tokenize the statement and produce a vector of tokens with location information
pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> { pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithSpan>, TokenizerError> {
let mut tokens: Vec<TokenWithLocation> = vec![]; let mut tokens: Vec<TokenWithSpan> = vec![];
self.tokenize_with_location_into_buf(&mut tokens) self.tokenize_with_location_into_buf(&mut tokens)
.map(|_| tokens) .map(|_| tokens)
} }
@ -726,7 +730,7 @@ impl<'a> Tokenizer<'a> {
/// If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error. /// If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error.
pub fn tokenize_with_location_into_buf( pub fn tokenize_with_location_into_buf(
&mut self, &mut self,
buf: &mut Vec<TokenWithLocation>, buf: &mut Vec<TokenWithSpan>,
) -> Result<(), TokenizerError> { ) -> Result<(), TokenizerError> {
let mut state = State { let mut state = State {
peekable: self.query.chars().peekable(), peekable: self.query.chars().peekable(),
@ -738,7 +742,7 @@ impl<'a> Tokenizer<'a> {
while let Some(token) = self.next_token(&mut state)? { while let Some(token) = self.next_token(&mut state)? {
let span = location.span_to(state.location()); let span = location.span_to(state.location());
buf.push(TokenWithLocation { token, span }); buf.push(TokenWithSpan { token, span });
location = state.location(); location = state.location();
} }
@ -2751,25 +2755,25 @@ mod tests {
.tokenize_with_location() .tokenize_with_location()
.unwrap(); .unwrap();
let expected = vec![ let expected = vec![
TokenWithLocation::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()), TokenWithSpan::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()),
TokenWithLocation::at( TokenWithSpan::at(
Token::Whitespace(Whitespace::Space), Token::Whitespace(Whitespace::Space),
(1, 7).into(), (1, 7).into(),
(1, 8).into(), (1, 8).into(),
), ),
TokenWithLocation::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()), TokenWithSpan::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()),
TokenWithLocation::at(Token::Comma, (1, 9).into(), (1, 10).into()), TokenWithSpan::at(Token::Comma, (1, 9).into(), (1, 10).into()),
TokenWithLocation::at( TokenWithSpan::at(
Token::Whitespace(Whitespace::Newline), Token::Whitespace(Whitespace::Newline),
(1, 10).into(), (1, 10).into(),
(2, 1).into(), (2, 1).into(),
), ),
TokenWithLocation::at( TokenWithSpan::at(
Token::Whitespace(Whitespace::Space), Token::Whitespace(Whitespace::Space),
(2, 1).into(), (2, 1).into(),
(2, 2).into(), (2, 2).into(),
), ),
TokenWithLocation::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()), TokenWithSpan::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()),
]; ];
compare(expected, tokens); compare(expected, tokens);
} }