mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-15 19:50:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Improve tokenizer error (#347)

Browse source 
* Improve tokenizer error

Signed-off-by: koushiro <koushiro.cqx@gmail.com>

* Add test for TokenizerError

Signed-off-by: koushiro <koushiro.cqx@gmail.com>
This commit is contained in:
Qinxuan Chen 2021-09-10 23:33:13 +08:00 • committed by GitHub
parent 10ec257e21
commit 3d6637c4f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 38 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

47
src/tokenizer.rs
View file

@ -287,6 +287,19 @@ pub struct TokenizerError {
pub col: u64,
}
impl fmt::Display for TokenizerError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"{} at Line: {}, Column {}",
self.message, self.line, self.col
)
}
}
#[cfg(feature = "std")]
impl std::error::Error for TokenizerError {}
/// SQL Tokenizer
pub struct Tokenizer<'a> {
dialect: &'a dyn Dialect,
@ -408,10 +421,10 @@ impl<'a> Tokenizer<'a> {
if chars.next() == Some(quote_end) {
Ok(Some(Token::make_word(&s, Some(quote_start))))
} else {
self.tokenizer_error(
format!("Expected close delimiter '{}' before EOF.", quote_end)
.as_str(),
)
self.tokenizer_error(format!(
"Expected close delimiter '{}' before EOF.",
quote_end
))
}
}
// numbers and period
@ -589,9 +602,9 @@ impl<'a> Tokenizer<'a> {
}
}
fn tokenizer_error<R>(&self, message: &str) -> Result<R, TokenizerError> {
fn tokenizer_error<R>(&self, message: impl Into<String>) -> Result<R, TokenizerError> {
Err(TokenizerError {
message: message.to_string(),
message: message.into(),
col: self.col,
line: self.line,
})
@ -703,9 +716,23 @@ fn peeking_take_while(
#[cfg(test)]
mod tests {
use super::super::dialect::GenericDialect;
use super::super::dialect::MsSqlDialect;
use super::*;
use crate::dialect::{GenericDialect, MsSqlDialect};
#[test]
fn tokenizer_error_impl() {
let err = TokenizerError {
message: "test".into(),
line: 1,
col: 1,
};
#[cfg(feature = "std")]
{
use std::error::Error;
assert!(err.source().is_none());
}
assert_eq!(err.to_string(), "test at Line: 1, Column 1");
}
#[test]
fn tokenize_select_1() {
@ -930,7 +957,7 @@ mod tests {
let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();
println!("tokens: {:#?}", tokens);
// println!("tokens: {:#?}", tokens);
let expected = vec![
Token::Whitespace(Whitespace::Newline),
Token::Char('م'),
@ -977,7 +1004,7 @@ mod tests {
let dialect = GenericDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();
println!("tokens: {:#?}", tokens);
// println!("tokens: {:#?}", tokens);
let expected = vec![
Token::Whitespace(Whitespace::Newline),
Token::Whitespace(Whitespace::Newline),