mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-19 13:40:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improve tokenizer error (#347)

Browse source 
* Improve tokenizer error

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* Add test for TokenizerError

Signed-off-by: koushiro <koushiro.cqx@gmail.com>
This commit is contained in:
Qinxuan Chen 2021-09-10 23:33:13 +08:00 • committed by GitHub
parent 10ec257e21
commit 3d6637c4f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/parser.rs
View file

@ -68,10 +68,7 @@ use IsLateral::*;
impl From<TokenizerError> for ParserError { impl From<TokenizerError> for ParserError {
fn from(e: TokenizerError) -> Self { fn from(e: TokenizerError) -> Self {
ParserError::TokenizerError(format!( ParserError::TokenizerError(e.to_string())
"{} at Line: {}, Column {}",
e.message, e.line, e.col
))
} }
} }

47
src/tokenizer.rs
View file

@ -287,6 +287,19 @@ pub struct TokenizerError {
pub col: u64, pub col: u64,
} }
impl fmt::Display for TokenizerError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} at Line: {}, Column {}",
self.message, self.line, self.col
)
}
}
#[cfg(feature = "std")]
impl std::error::Error for TokenizerError {}
/// SQL Tokenizer /// SQL Tokenizer
pub struct Tokenizer<'a> { pub struct Tokenizer<'a> {
dialect: &'a dyn Dialect, dialect: &'a dyn Dialect,
@ -408,10 +421,10 @@ impl<'a> Tokenizer<'a> {
if chars.next() == Some(quote_end) { if chars.next() == Some(quote_end) {
Ok(Some(Token::make_word(&s, Some(quote_start)))) Ok(Some(Token::make_word(&s, Some(quote_start))))
} else { } else {
self.tokenizer_error( self.tokenizer_error(format!(
format!("Expected close delimiter '{}' before EOF.", quote_end) "Expected close delimiter '{}' before EOF.",
.as_str(), quote_end
) ))
} }
} }
// numbers and period // numbers and period
@ -589,9 +602,9 @@ impl<'a> Tokenizer<'a> {
} }
} }
fn tokenizer_error<R>(&self, message: &str) -> Result<R, TokenizerError> { fn tokenizer_error<R>(&self, message: impl Into<String>) -> Result<R, TokenizerError> {
Err(TokenizerError { Err(TokenizerError {
message: message.to_string(), message: message.into(),
col: self.col, col: self.col,
line: self.line, line: self.line,
}) })
@ -703,9 +716,23 @@ fn peeking_take_while(
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::super::dialect::GenericDialect;
use super::super::dialect::MsSqlDialect;
use super::*; use super::*;
use crate::dialect::{GenericDialect, MsSqlDialect};
#[test]
fn tokenizer_error_impl() {
let err = TokenizerError {
message: "test".into(),
line: 1,
col: 1,
};
#[cfg(feature = "std")]
{
use std::error::Error;
assert!(err.source().is_none());
}
assert_eq!(err.to_string(), "test at Line: 1, Column 1");
}
#[test] #[test]
fn tokenize_select_1() { fn tokenize_select_1() {
@ -930,7 +957,7 @@ mod tests {
let dialect = GenericDialect {}; let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
println!("tokens: {:#?}", tokens); // println!("tokens: {:#?}", tokens);
let expected = vec![ let expected = vec![
Token::Whitespace(Whitespace::Newline), Token::Whitespace(Whitespace::Newline),
Token::Char('م'), Token::Char('م'),
@ -977,7 +1004,7 @@ mod tests {
let dialect = GenericDialect {}; let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
println!("tokens: {:#?}", tokens); // println!("tokens: {:#?}", tokens);
let expected = vec![ let expected = vec![
Token::Whitespace(Whitespace::Newline), Token::Whitespace(Whitespace::Newline),
Token::Whitespace(Whitespace::Newline), Token::Whitespace(Whitespace::Newline),