mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-25 00:14:06 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Rename TokenWithLocation to TokenWithSpan, in backwards compatible way (#1562)

Browse source
This commit is contained in:
Andrew Lamb 2024-11-30 07:55:21 -05:00 • committed by GitHub
parent 92c6e7f79b
commit a134910a36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 67 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/ast/helpers/attached_token.rs
View file

@ -19,7 +19,7 @@ use core::cmp::{Eq, Ord, Ordering, PartialEq, PartialOrd};
use core::fmt::{self, Debug, Formatter};
use core::hash::{Hash, Hasher};
use crate::tokenizer::{Token, TokenWithLocation};
use crate::tokenizer::{Token, TokenWithSpan};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
@ -33,11 +33,11 @@ use sqlparser_derive::{Visit, VisitMut};
#[derive(Clone)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct AttachedToken(pub TokenWithLocation);
pub struct AttachedToken(pub TokenWithSpan);
impl AttachedToken {
pub fn empty() -> Self {
AttachedToken(TokenWithLocation::wrap(Token::EOF))
AttachedToken(TokenWithSpan::wrap(Token::EOF))
}
}
@ -75,8 +75,8 @@ impl Hash for AttachedToken {
}
}
impl From<TokenWithLocation> for AttachedToken {
fn from(value: TokenWithLocation) -> Self {
impl From<TokenWithSpan> for AttachedToken {
fn from(value: TokenWithSpan) -> Self {
AttachedToken(value)
}
}

4
src/ast/query.rs
View file

@ -27,7 +27,7 @@ use sqlparser_derive::{Visit, VisitMut};
use crate::{
ast::*,
tokenizer::{Token, TokenWithLocation},
tokenizer::{Token, TokenWithSpan},
};
/// The most complete variant of a `SELECT` query expression, optionally
@ -643,7 +643,7 @@ pub struct WildcardAdditionalOptions {
impl Default for WildcardAdditionalOptions {
fn default() -> Self {
Self {
wildcard_token: TokenWithLocation::wrap(Token::Mul).into(),
wildcard_token: TokenWithSpan::wrap(Token::Mul).into(),
opt_ilike: None,
opt_exclude: None,
opt_except: None,

66
src/parser/mod.rs
View file

@ -265,7 +265,7 @@ enum ParserState {
}
pub struct Parser<'a> {
tokens: Vec<TokenWithLocation>,
tokens: Vec<TokenWithSpan>,
/// The index of the first unprocessed token in [`Parser::tokens`].
index: usize,
/// The current state of the parser.
@ -359,7 +359,7 @@ impl<'a> Parser<'a> {
}
/// Reset this parser to parse the specified token stream
pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithLocation>) -> Self {
pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithSpan>) -> Self {
self.tokens = tokens;
self.index = 0;
self
@ -368,9 +368,9 @@ impl<'a> Parser<'a> {
/// Reset this parser state to parse the specified tokens
pub fn with_tokens(self, tokens: Vec<Token>) -> Self {
// Put in dummy locations
let tokens_with_locations: Vec<TokenWithLocation> = tokens
let tokens_with_locations: Vec<TokenWithSpan> = tokens
.into_iter()
.map(|token| TokenWithLocation {
.map(|token| TokenWithSpan {
token,
span: Span::empty(),
})
@ -1147,7 +1147,7 @@ impl<'a> Parser<'a> {
match self.peek_token().token {
Token::LParen | Token::Period => {
let mut id_parts: Vec<Ident> = vec![w.to_ident(w_span)];
let mut ending_wildcard: Option<TokenWithLocation> = None;
let mut ending_wildcard: Option<TokenWithSpan> = None;
while self.consume_token(&Token::Period) {
let next_token = self.next_token();
match next_token.token {
@ -3273,7 +3273,7 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file)
pub fn peek_token(&self) -> TokenWithLocation {
pub fn peek_token(&self) -> TokenWithSpan {
self.peek_nth_token(0)
}
@ -3308,19 +3308,19 @@ impl<'a> Parser<'a> {
/// yet been processed.
///
/// See [`Self::peek_token`] for an example.
pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithLocation; N] {
pub fn peek_tokens_with_location<const N: usize>(&self) -> [TokenWithSpan; N] {
let mut index = self.index;
core::array::from_fn(|_| loop {
let token = self.tokens.get(index);
index += 1;
if let Some(TokenWithLocation {
if let Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) = token
{
continue;
}
break token.cloned().unwrap_or(TokenWithLocation {
break token.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
});
@ -3328,18 +3328,18 @@ impl<'a> Parser<'a> {
}
/// Return nth non-whitespace token that has not yet been processed
pub fn peek_nth_token(&self, mut n: usize) -> TokenWithLocation {
pub fn peek_nth_token(&self, mut n: usize) -> TokenWithSpan {
let mut index = self.index;
loop {
index += 1;
match self.tokens.get(index - 1) {
Some(TokenWithLocation {
Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) => continue,
non_whitespace => {
if n == 0 {
return non_whitespace.cloned().unwrap_or(TokenWithLocation {
return non_whitespace.cloned().unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
});
@ -3352,16 +3352,16 @@ impl<'a> Parser<'a> {
/// Return the first token, possibly whitespace, that has not yet been processed
/// (or None if reached end-of-file).
pub fn peek_token_no_skip(&self) -> TokenWithLocation {
pub fn peek_token_no_skip(&self) -> TokenWithSpan {
self.peek_nth_token_no_skip(0)
}
/// Return nth token, possibly whitespace, that has not yet been processed.
pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithLocation {
pub fn peek_nth_token_no_skip(&self, n: usize) -> TokenWithSpan {
self.tokens
.get(self.index + n)
.cloned()
.unwrap_or(TokenWithLocation {
.unwrap_or(TokenWithSpan {
token: Token::EOF,
span: Span::empty(),
})
@ -3378,25 +3378,25 @@ impl<'a> Parser<'a> {
/// Return the first non-whitespace token that has not yet been processed
/// (or None if reached end-of-file) and mark it as processed. OK to call
/// repeatedly after reaching EOF.
pub fn next_token(&mut self) -> TokenWithLocation {
pub fn next_token(&mut self) -> TokenWithSpan {
loop {
self.index += 1;
match self.tokens.get(self.index - 1) {
Some(TokenWithLocation {
Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) => continue,
token => {
return token
.cloned()
.unwrap_or_else(|| TokenWithLocation::wrap(Token::EOF))
.unwrap_or_else(|| TokenWithSpan::wrap(Token::EOF))
}
}
}
}
/// Return the first unprocessed token, possibly whitespace.
pub fn next_token_no_skip(&mut self) -> Option<&TokenWithLocation> {
pub fn next_token_no_skip(&mut self) -> Option<&TokenWithSpan> {
self.index += 1;
self.tokens.get(self.index - 1)
}
@ -3408,7 +3408,7 @@ impl<'a> Parser<'a> {
loop {
assert!(self.index > 0);
self.index -= 1;
if let Some(TokenWithLocation {
if let Some(TokenWithSpan {
token: Token::Whitespace(_),
span: _,
}) = self.tokens.get(self.index)
@ -3420,7 +3420,7 @@ impl<'a> Parser<'a> {
}
/// Report `found` was encountered instead of `expected`
pub fn expected<T>(&self, expected: &str, found: TokenWithLocation) -> Result<T, ParserError> {
pub fn expected<T>(&self, expected: &str, found: TokenWithSpan) -> Result<T, ParserError> {
parser_err!(
format!("Expected: {expected}, found: {found}"),
found.span.start
@ -3435,7 +3435,7 @@ impl<'a> Parser<'a> {
}
#[must_use]
pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithLocation> {
pub fn parse_keyword_token(&mut self, expected: Keyword) -> Option<TokenWithSpan> {
match self.peek_token().token {
Token::Word(w) if expected == w.keyword => Some(self.next_token()),
_ => None,
@ -3524,7 +3524,7 @@ impl<'a> Parser<'a> {
/// If the current token is the `expected` keyword, consume the token.
/// Otherwise, return an error.
pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithLocation, ParserError> {
pub fn expect_keyword(&mut self, expected: Keyword) -> Result<TokenWithSpan, ParserError> {
if let Some(token) = self.parse_keyword_token(expected) {
Ok(token)
} else {
@ -3568,7 +3568,7 @@ impl<'a> Parser<'a> {
}
/// Bail out if the current token is not an expected keyword, or consume it if it is
pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithLocation, ParserError> {
pub fn expect_token(&mut self, expected: &Token) -> Result<TokenWithSpan, ParserError> {
if self.peek_token() == *expected {
Ok(self.next_token())
} else {
@ -4107,7 +4107,7 @@ impl<'a> Parser<'a> {
Keyword::ARCHIVE => Ok(Some(CreateFunctionUsing::Archive(uri))),
_ => self.expected(
"JAR, FILE or ARCHIVE, got {:?}",
TokenWithLocation::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
TokenWithSpan::wrap(Token::make_keyword(format!("{keyword:?}").as_str())),
),
}
}
@ -6832,7 +6832,7 @@ impl<'a> Parser<'a> {
if let Some(name) = name {
return self.expected(
"FULLTEXT or SPATIAL option without constraint name",
TokenWithLocation {
TokenWithSpan {
token: Token::make_keyword(&name.to_string()),
span: next_token.span,
},
@ -7808,7 +7808,7 @@ impl<'a> Parser<'a> {
Some('\'') => Ok(Value::SingleQuotedString(w.value)),
_ => self.expected(
"A value?",
TokenWithLocation {
TokenWithSpan {
token: Token::Word(w),
span,
},
@ -7816,7 +7816,7 @@ impl<'a> Parser<'a> {
},
_ => self.expected(
"a concrete value",
TokenWithLocation {
TokenWithSpan {
token: Token::Word(w),
span,
},
@ -7878,7 +7878,7 @@ impl<'a> Parser<'a> {
}
unexpected => self.expected(
"a value",
TokenWithLocation {
TokenWithSpan {
token: unexpected,
span,
},
@ -7927,7 +7927,7 @@ impl<'a> Parser<'a> {
Token::HexStringLiteral(ref s) => Ok(Value::HexStringLiteral(s.to_string())),
unexpected => self.expected(
"a string value",
TokenWithLocation {
TokenWithSpan {
token: unexpected,
span,
},
@ -8618,7 +8618,7 @@ impl<'a> Parser<'a> {
let token = self
.next_token_no_skip()
.cloned()
.unwrap_or(TokenWithLocation::wrap(Token::EOF));
.unwrap_or(TokenWithSpan::wrap(Token::EOF));
requires_whitespace = match token.token {
Token::Word(next_word) if next_word.quote_style.is_none() => {
ident.value.push_str(&next_word.value);
@ -11683,7 +11683,7 @@ impl<'a> Parser<'a> {
/// If it is not possible to parse it, will return an option.
pub fn parse_wildcard_additional_options(
&mut self,
wildcard_token: TokenWithLocation,
wildcard_token: TokenWithSpan,
) -> Result<WildcardAdditionalOptions, ParserError> {
let opt_ilike = if dialect_of!(self is GenericDialect | SnowflakeDialect) {
self.parse_optional_select_item_ilike()?
@ -12708,7 +12708,7 @@ impl<'a> Parser<'a> {
}
/// Consume the parser and return its underlying token buffer
pub fn into_tokens(self) -> Vec<TokenWithLocation> {
pub fn into_tokens(self) -> Vec<TokenWithSpan> {
self.tokens
}

50
src/tokenizer.rs
View file

@ -521,42 +521,46 @@ impl Span {
}
}
/// Backwards compatibility struct for [`TokenWithSpan`]
#[deprecated(since = "0.53.0", note = "please use `TokenWithSpan` instead")]
pub type TokenWithLocation = TokenWithSpan;
/// A [Token] with [Location] attached to it
#[derive(Debug, Clone, Hash, Ord, PartialOrd, Eq, PartialEq)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
pub struct TokenWithLocation {
pub struct TokenWithSpan {
pub token: Token,
pub span: Span,
}
impl TokenWithLocation {
pub fn new(token: Token, span: Span) -> TokenWithLocation {
TokenWithLocation { token, span }
impl TokenWithSpan {
pub fn new(token: Token, span: Span) -> TokenWithSpan {
TokenWithSpan { token, span }
}
pub fn wrap(token: Token) -> TokenWithLocation {
TokenWithLocation::new(token, Span::empty())
pub fn wrap(token: Token) -> TokenWithSpan {
TokenWithSpan::new(token, Span::empty())
}
pub fn at(token: Token, start: Location, end: Location) -> TokenWithLocation {
TokenWithLocation::new(token, Span::new(start, end))
pub fn at(token: Token, start: Location, end: Location) -> TokenWithSpan {
TokenWithSpan::new(token, Span::new(start, end))
}
}
impl PartialEq<Token> for TokenWithLocation {
impl PartialEq<Token> for TokenWithSpan {
fn eq(&self, other: &Token) -> bool {
&self.token == other
}
}
impl PartialEq<TokenWithLocation> for Token {
fn eq(&self, other: &TokenWithLocation) -> bool {
impl PartialEq<TokenWithSpan> for Token {
fn eq(&self, other: &TokenWithSpan) -> bool {
self == &other.token
}
}
impl fmt::Display for TokenWithLocation {
impl fmt::Display for TokenWithSpan {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.token.fmt(f)
}
@ -716,8 +720,8 @@ impl<'a> Tokenizer<'a> {
}
/// Tokenize the statement and produce a vector of tokens with location information
pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> {
let mut tokens: Vec<TokenWithLocation> = vec![];
pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithSpan>, TokenizerError> {
let mut tokens: Vec<TokenWithSpan> = vec![];
self.tokenize_with_location_into_buf(&mut tokens)
.map(|_| tokens)
}
@ -726,7 +730,7 @@ impl<'a> Tokenizer<'a> {
/// If an error is thrown, the buffer will contain all tokens that were successfully parsed before the error.
pub fn tokenize_with_location_into_buf(
&mut self,
buf: &mut Vec<TokenWithLocation>,
buf: &mut Vec<TokenWithSpan>,
) -> Result<(), TokenizerError> {
let mut state = State {
peekable: self.query.chars().peekable(),
@ -738,7 +742,7 @@ impl<'a> Tokenizer<'a> {
while let Some(token) = self.next_token(&mut state)? {
let span = location.span_to(state.location());
buf.push(TokenWithLocation { token, span });
buf.push(TokenWithSpan { token, span });
location = state.location();
}
@ -2751,25 +2755,25 @@ mod tests {
.tokenize_with_location()
.unwrap();
let expected = vec![
TokenWithLocation::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()),
TokenWithLocation::at(
TokenWithSpan::at(Token::make_keyword("SELECT"), (1, 1).into(), (1, 7).into()),
TokenWithSpan::at(
Token::Whitespace(Whitespace::Space),
(1, 7).into(),
(1, 8).into(),
),
TokenWithLocation::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()),
TokenWithLocation::at(Token::Comma, (1, 9).into(), (1, 10).into()),
TokenWithLocation::at(
TokenWithSpan::at(Token::make_word("a", None), (1, 8).into(), (1, 9).into()),
TokenWithSpan::at(Token::Comma, (1, 9).into(), (1, 10).into()),
TokenWithSpan::at(
Token::Whitespace(Whitespace::Newline),
(1, 10).into(),
(2, 1).into(),
),
TokenWithLocation::at(
TokenWithSpan::at(
Token::Whitespace(Whitespace::Space),
(2, 1).into(),
(2, 2).into(),
),
TokenWithLocation::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()),
TokenWithSpan::at(Token::make_word("b", None), (2, 2).into(), (2, 3).into()),
];
compare(expected, tokens);
}