mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-23 23:44:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Refactoring use of generics, added peek_token

Browse source
This commit is contained in:
Andy Grove 2018-09-01 08:11:26 -06:00
parent eed0261297
commit e6e9c8d2cc
5 changed files with 56 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/tokenizer.rs
View file

@ -47,21 +47,24 @@ pub enum SQLToken<T: Debug + PartialEq> {
Custom(T)
}
pub trait SQLTokenizer<S, TE>
where S: Debug + PartialEq {
pub trait SQLTokenizer<TokenType>
where TokenType: Debug + PartialEq {
/// return a reference to the next token but do not advance the index
fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
/// return a reference to the next token and advance the index
fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<S>>, TokenizerError<TE>>;
fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
}
pub fn tokenize<S,TE>(sql: &str, tokenizer: &mut SQLTokenizer<S,TE>) -> Result<Vec<SQLToken<S>>, TokenizerError<TE>>
where S: Debug + PartialEq
pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
where TokenType: Debug + PartialEq
{
let mut peekable = sql.chars().peekable();
let mut tokens : Vec<SQLToken<S>> = vec![];
let mut tokens : Vec<SQLToken<TokenType>> = vec![];
loop {
match tokenizer.next_token(&mut peekable)? {