mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-20 06:00:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Refactoring

Browse source
This commit is contained in:
Andy Grove 2018-09-01 12:59:18 -06:00
parent 2ddc9f7a49
commit 7ff4133327
5 changed files with 108 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

56
src/tokenizer.rs
View file

@ -1,7 +1,7 @@
use std::cmp::PartialEq;
use std::fmt::Debug;
use std::iter::Peekable;
use std::str::Chars;
//use std::iter::Peekable;
//use std::str::Chars;
#[derive(Debug)]
pub struct Position {
@ -54,31 +54,35 @@ pub trait SQLTokenizer<TokenType>
fn precedence(&self, token: &SQLToken<TokenType>) -> usize;
/// return a reference to the next token but do not advance the index
fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
fn peek_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
/// return a reference to the next token and advance the index
fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
fn peek_char(&mut self) -> Option<&char>;
fn next_char(&mut self) -> Option<&char>;
}
pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
where TokenType: Debug + PartialEq
{
let mut peekable = sql.chars().peekable();
let mut tokens : Vec<SQLToken<TokenType>> = vec![];
loop {
match tokenizer.next_token(&mut peekable)? {
Some(SQLToken::Whitespace(_)) => { /* ignore */ },
Some(token) => {
println!("Token: {:?}", token);
tokens.push(token)
},
None => break
}
}
Ok(tokens)
}
//
//pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
// where TokenType: Debug + PartialEq
// {
//
// let mut peekable = sql.chars().peekable();
//
// let mut tokens : Vec<SQLToken<TokenType>> = vec![];
//
// loop {
// match tokenizer.next_token(&mut peekable)? {
// Some(SQLToken::Whitespace(_)) => { /* ignore */ },
// Some(token) => {
// println!("Token: {:?}", token);
// tokens.push(token)
// },
// None => break
// }
// }
//
// Ok(tokens)
//}