Refactoring use of generics, added peek_token

examples/acme_parser.rs

@@ -32,9 +32,13 @@ struct AcmeTokenizer {
 }
 
 /// The ACME tokenizer looks for the factorial operator `!!` but delegates everything else
-impl SQLTokenizer<AcmeToken, AcmeTokenizerError> for AcmeTokenizer {
+impl SQLTokenizer<AcmeToken> for AcmeTokenizer {
 
-    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeTokenizerError>> {
+    fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
+        unimplemented!()
+    }
+
+    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
         match chars.peek() {
             Some(&ch) => match ch {
                 '!' => {

src/ansi/parser.rs

@@ -6,18 +6,30 @@ use std::str::Chars;
 use super::super::tokenizer::*;
 use super::super::parser::*;
 
-pub struct ANSISQLParser {
+pub struct ANSISQLParser<'a, TokenType> {
-
+    chars: Peekable<Chars<'a>>,
+    tokenizer: SQLTokenizer<TokenType>
 }
 
-impl<S,TE> SQLParser<S,TE> for ANSISQLParser
+impl<'a, TokenType, ExprType> SQLParser<TokenType, ExprType> for ANSISQLParser<'a, TokenType>
-    where S: Debug + PartialEq {
+    where TokenType: Debug + PartialEq, ExprType: Debug + PartialEq {
 
-    fn parse_prefix(&mut self) -> Result<Box<SQLExpr<S>>, ParserError<S, TE>> {
+    fn parse_prefix(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> {
-        unimplemented!()
+
+        match self.tokenizer.peek_token(&mut self.chars)? {
+            Some(SQLToken::Keyword(ref k)) => match k.to_uppercase().as_ref() {
+                "INSERT" => unimplemented!(),
+                "UPDATE" => unimplemented!(),
+                "DELETE" => unimplemented!(),
+                "SELECT" => unimplemented!(),
+                "CREATE" => unimplemented!(),
+                _ => unimplemented!()
+            },
+            _ => unimplemented!()
+        }
     }
 
-    fn parse_infix(&mut self, left: SQLExpr<S>) -> Result<Option<Box<SQLExpr<S>>>, ParserError<S, TE>> {
+    fn parse_infix(&mut self, left: SQLExpr<ExprType>) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>> {
         unimplemented!()
     }
 }

src/ansi/tokenizer.rs

@@ -7,10 +7,14 @@ use super::super::tokenizer::*;
 
 pub struct ANSISQLTokenizer {}
 
-impl<S,TE> SQLTokenizer<S,TE> for ANSISQLTokenizer
+impl<TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer
-    where S: Debug + PartialEq {
+    where TokenType: Debug + PartialEq {
 
-    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<S>>, TokenizerError<TE>> {
+    fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
+        unimplemented!()
+    }
+
+    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
         match chars.next() {
             Some(ch) => match ch {
                 ' ' | '\t' | '\n' => Ok(Some(SQLToken::Whitespace(ch))),

src/parser.rs

@@ -70,48 +70,47 @@ pub enum SQLOperator<T> {
 
 /// SQL Expressions
 #[derive(Debug)]
-pub enum SQLExpr<T> {
+pub enum SQLExpr<ExprType> {
     /// Identifier e.g. table name or column name
     Identifier(String),
     /// Literal value
     Literal(String),
     /// Binary expression e.g. `1 + 2` or `fname LIKE "A%"`
-    Binary(Box<SQLExpr<T>>, SQLOperator<T>, Box<SQLExpr<T>>),
+    Binary(Box<SQLExpr<ExprType>>, SQLOperator<ExprType>, Box<SQLExpr<ExprType>>),
     /// Function invocation with function name and list of argument expressions
-    FunctionCall(String, Vec<SQLExpr<T>>),
+    FunctionCall(String, Vec<SQLExpr<ExprType>>),
     Insert,
     Update,
     Delete,
     Select,
     CreateTable,
     /// Custom expression (vendor-specific)
-    Custom(T)
+    Custom(ExprType)
 }
 
 #[derive(Debug)]
-pub enum ParserError<S, PE>
+pub enum ParserError<TokenType>
-    where S: Debug + PartialEq {
+    where TokenType: Debug + PartialEq {
-
+    WrongToken { expected: Vec<SQLToken<TokenType>>, actual: SQLToken<TokenType>, line: usize, col: usize },
-    WrongToken { expected: Vec<SQLToken<S>>, actual: SQLToken<S>, line: usize, col: usize },
+    Custom(String)
-    Custom(PE)
 }
 
-impl<S, TE> From<TokenizerError<TE>> for ParserError<S, TE>
+impl<TokenType> From<TokenizerError<TokenType>> for ParserError<TokenType>
-    where S: Debug + PartialEq {
+    where TokenType: Debug + PartialEq {
 
-    fn from(_: TokenizerError<TE>) -> Self {
+    fn from(_: TokenizerError<TokenType>) -> Self {
         unimplemented!()
     }
 }
 
 
-pub trait SQLParser<S, PE>
+pub trait SQLParser<TokenType, ExprType>
-    where S: Debug + PartialEq {
+    where TokenType: Debug + PartialEq, ExprType: Debug + PartialEq {
 
     /// parse the prefix and stop once an infix operator is reached
-    fn parse_prefix(&mut self) -> Result<Box<SQLExpr<S>>, ParserError<S, PE>> ;
+    fn parse_prefix(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> ;
     /// parse the next infix expression, returning None if the precedence has changed
-    fn parse_infix(&mut self, left: SQLExpr<S>) -> Result<Option<Box<SQLExpr<S>>>, ParserError<S, PE>>;
+    fn parse_infix(&mut self, left: SQLExpr<ExprType>) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>>;
 }
 
 //

src/tokenizer.rs

@@ -47,21 +47,24 @@ pub enum SQLToken<T: Debug + PartialEq> {
     Custom(T)
 }
 
-pub trait SQLTokenizer<S, TE>
+pub trait SQLTokenizer<TokenType>
-    where S: Debug + PartialEq {
+    where TokenType: Debug + PartialEq {
+
+    /// return a reference to the next token but do not advance the index
+    fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
 
     /// return a reference to the next token and advance the index
-    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<S>>, TokenizerError<TE>>;
+    fn next_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
 }
 
 
-pub fn tokenize<S,TE>(sql: &str, tokenizer: &mut SQLTokenizer<S,TE>) -> Result<Vec<SQLToken<S>>, TokenizerError<TE>>
+pub fn tokenize<TokenType>(sql: &str, tokenizer: &mut SQLTokenizer<TokenType>) -> Result<Vec<SQLToken<TokenType>>, TokenizerError<TokenType>>
-    where S: Debug + PartialEq
+    where TokenType: Debug + PartialEq
     {
 
     let mut peekable = sql.chars().peekable();
 
-    let mut tokens : Vec<SQLToken<S>> = vec![];
+    let mut tokens : Vec<SQLToken<TokenType>> = vec![];
 
     loop {
         match tokenizer.next_token(&mut peekable)? {