mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-31 11:17:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add line and column number to TokenizerError (#194)

Browse source 
Addresses https://github.com/andygrove/sqlparser-rs/issues/179 for tokenize errors
This commit is contained in:
Daniël Heres 2020-06-10 08:15:44 +02:00 • committed by GitHub
parent 10b0b7f884
commit d842f495db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 34 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

54
src/tokenizer.rs
View file

@ -217,7 +217,11 @@ impl fmt::Display for Whitespace {
/// Tokenizer error
#[derive(Debug, PartialEq)]
pub struct TokenizerError(String);
pub struct TokenizerError {
pub message: String,
pub line: u64,
pub col: u64,
}
/// SQL Tokenizer
pub struct Tokenizer<'a> {
@ -331,10 +335,10 @@ impl<'a> Tokenizer<'a> {
if chars.next() == Some(quote_end) {
Ok(Some(Token::make_word(&s, Some(quote_start))))
} else {
Err(TokenizerError(format!(
"Expected close delimiter '{}' before EOF.",
quote_end
)))
self.tokenizer_error(
format!("Expected close delimiter '{}' before EOF.", quote_end)
.as_str(),
)
}
}
// numbers
@ -395,10 +399,7 @@ impl<'a> Tokenizer<'a> {
chars.next(); // consume
match chars.peek() {
Some('=') => self.consume_and_return(chars, Token::Neq),
_ => Err(TokenizerError(format!(
"Tokenizer Error at Line: {}, Col: {}",
self.line, self.col
))),
_ => self.tokenizer_error("Expected to see '=' after '!' character"),
}
}
'<' => {
@ -437,6 +438,14 @@ impl<'a> Tokenizer<'a> {
}
}
fn tokenizer_error<R>(&self, message: &str) -> Result<R, TokenizerError> {
Err(TokenizerError {
message: message.to_string(),
col: self.col,
line: self.line,
})
}
/// Tokenize an identifier or keyword, after the first char is already consumed.
fn tokenize_word(&self, first_char: char, chars: &mut Peekable<Chars<'_>>) -> String {
let mut s = first_char.to_string();
@ -471,10 +480,7 @@ impl<'a> Tokenizer<'a> {
}
}
}
Err(TokenizerError(format!(
"Unterminated string literal at Line: {}, Col: {}",
self.line, self.col
)))
self.tokenizer_error("Unterminated string literal")
}
fn tokenize_multiline_comment(
@ -499,11 +505,7 @@ impl<'a> Tokenizer<'a> {
s.push(ch);
}
}
None => {
break Err(TokenizerError(
"Unexpected EOF while in a multi-line comment".to_string(),
));
}
None => break self.tokenizer_error("Unexpected EOF while in a multi-line comment"),
}
}
}
@ -720,9 +722,11 @@ mod tests {
let mut tokenizer = Tokenizer::new(&dialect, &sql);
assert_eq!(
tokenizer.tokenize(),
Err(TokenizerError(
"Unterminated string literal at Line: 1, Col: 8".to_string()
))
Err(TokenizerError {
message: "Unterminated string literal".to_string(),
line: 1,
col: 8
})
);
}
@ -843,9 +847,11 @@ mod tests {
let mut tokenizer = Tokenizer::new(&dialect, &sql);
assert_eq!(
tokenizer.tokenize(),
Err(TokenizerError(
"Expected close delimiter '\"' before EOF.".to_string(),
))
Err(TokenizerError {
message: "Expected close delimiter '\"' before EOF.".to_string(),
line: 1,
col: 1
})
);
}