Roughing out pratt parser logic

examples/acme_parser.rs

@@ -7,24 +7,21 @@ use datafusion_sql::ansi::tokenizer::ANSISQLTokenizer;
 use datafusion_sql::tokenizer::*;
 use datafusion_sql::parser::*;
 
-///
 /// This example demonstrates building a custom ACME parser that extends the generic parser
-/// by adding support for a factorial operator !!
+/// by adding support for a factorial expression `!! expr`.
-///
 
+/// Custom SQLToken
 #[derive(Debug,PartialEq)]
 enum AcmeToken {
-    /// Factorial operator `!!`
+    /// Factorial token `!!`
     Factorial
 }
 
+/// Custom SQLExpr
 #[derive(Debug)]
-enum AcmeOperator {
+enum AcmeExpr {
-    Factorial
+    /// Factorial expression
-}
+    Factorial(Box<SQLExpr<AcmeExpr>>)
-
-#[derive(Debug)]
-enum AcmeTokenizerError {
 }
 
 struct AcmeTokenizer {
@@ -34,6 +31,10 @@ struct AcmeTokenizer {
 /// The ACME tokenizer looks for the factorial operator `!!` but delegates everything else
 impl SQLTokenizer<AcmeToken> for AcmeTokenizer {
 
+    fn precedence(&self, token: &SQLToken<AcmeToken>) -> usize {
+        unimplemented!()
+    }
+
     fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<AcmeToken>>, TokenizerError<AcmeToken>> {
         unimplemented!()
     }
@@ -53,7 +54,7 @@ impl SQLTokenizer<AcmeToken> for AcmeTokenizer {
                         },
                         None => Ok(Some(SQLToken::Not))
                     }
-                },
+                }
                 _ => self.generic.next_token(chars)
             }
             _ => self.generic.next_token(chars)
@@ -61,19 +62,47 @@ impl SQLTokenizer<AcmeToken> for AcmeTokenizer {
     }
 }
 
+struct AcmeParser<'a> {
+    chars: Peekable<Chars<'a>>
+}
+
+impl<'a> AcmeParser<'a> {
+
+    pub fn new(sql: &'a str) -> Self {
+        AcmeParser {
+            chars: sql.chars().peekable()
+        }
+    }
+}
+
+impl<'a> SQLParser<AcmeToken, AcmeExpr> for AcmeParser<'a> {
+
+    fn parse_prefix(&mut self) -> Result<Box<SQLExpr<AcmeExpr>>, ParserError<AcmeToken>> {
+        unimplemented!()
+    }
+
+    fn parse_infix(&mut self, left: &SQLExpr<AcmeExpr>, precedence: usize) -> Result<Option<Box<SQLExpr<AcmeExpr>>>, ParserError<AcmeToken>> {
+        unimplemented!()
+    }
+}
 
 
 fn main() {
 
     let sql = "1 + !! 5 * 2";
 
-    let mut acme_tokenizer = AcmeTokenizer {
+    let acme_parser = AcmeParser::new(sql);
-        generic: ANSISQLTokenizer { }
-    };
 
-    let tokens = tokenize(&sql, &mut acme_tokenizer).unwrap();
 
-    println!("tokens = {:?}", tokens);
+    //acme_parser
+
+//    let mut acme_tokenizer = AcmeTokenizer {
+//        generic: ANSISQLTokenizer { }
+//    };
+//
+//    let tokens = tokenize(&sql, &mut acme_tokenizer).unwrap();
+//
+//    println!("tokens = {:?}", tokens);
 
 
 

src/ansi/parser.rs

@@ -29,7 +29,7 @@ impl<'a, TokenType, ExprType> SQLParser<TokenType, ExprType> for ANSISQLParser<'
         }
     }
 
-    fn parse_infix(&mut self, left: SQLExpr<ExprType>) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>> {
+    fn parse_infix(&mut self, left: &SQLExpr<ExprType>, precedence: usize) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>> {
         unimplemented!()
     }
 }

src/ansi/tokenizer.rs

@@ -10,6 +10,10 @@ pub struct ANSISQLTokenizer {}
 impl<TokenType> SQLTokenizer<TokenType> for ANSISQLTokenizer
     where TokenType: Debug + PartialEq {
 
+    fn precedence(&self, token: &SQLToken<TokenType>) -> usize {
+        unimplemented!()
+    }
+
     fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>> {
         unimplemented!()
     }

src/parser.rs

@@ -1,5 +1,8 @@
 use std::cmp::PartialEq;
 use std::fmt::Debug;
+use std::rc::Rc;
+use std::str::Chars;
+use std::iter::Peekable;
 
 use super::tokenizer::*;
 
@@ -105,41 +108,47 @@ impl<TokenType> From<TokenizerError<TokenType>> for ParserError<TokenType>
 
 
 pub trait SQLParser<TokenType, ExprType>
-    where TokenType: Debug + PartialEq, ExprType: Debug + PartialEq {
+    where TokenType: Debug + PartialEq, ExprType: Debug {
 
     /// parse the prefix and stop once an infix operator is reached
     fn parse_prefix(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> ;
     /// parse the next infix expression, returning None if the precedence has changed
-    fn parse_infix(&mut self, left: SQLExpr<ExprType>) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>>;
+    fn parse_infix(&mut self, left: &SQLExpr<ExprType>, precedence: usize) -> Result<Option<Box<SQLExpr<ExprType>>>, ParserError<TokenType>>;
+}
+
+
+
+struct PrattParser<'a, TokenType, ExprType> {
+    chars: Peekable<Chars<'a>>,
+    tokenizer: Rc<SQLTokenizer<TokenType>>,
+    parser: SQLParser<TokenType, ExprType>
+}
+
+impl<'a, TokenType, ExprType> PrattParser<'a, TokenType, ExprType>
+    where TokenType: Debug + PartialEq, ExprType: Debug {
+
+    fn parse_expr(&mut self) -> Result<Box<SQLExpr<ExprType>>, ParserError<TokenType>> {
+
+        let precedence: usize = 0;
+
+        let mut expr = self.parser.parse_prefix()?;
+
+        while let Some(token) = self.tokenizer.peek_token(&mut self.chars)? {
+
+            let next_precedence = self.tokenizer.precedence(&token);
+
+            if precedence >= next_precedence {
+                break;
+            }
+
+            expr = self.parser.parse_infix(&expr, next_precedence)?.unwrap(); //TODO: fix me
+        }
+
+        Ok(expr)
+    }
+
 }
 
-//
-//
-//struct GenericParser {
-//    tokenizer: SQLTokenizer
-//}
-//
-//impl GenericParser {
-//
-//    fn parse_expr(&mut self, precedence: u8) -> Result<Box<SQLExpr>, ParserError> {
-//
-//        let mut expr = self.parse_prefix()?;
-//
-//        // loop while there are more tokens and until the precedence changes
-//        while let Some(token) = self.tokenizer.peek_token()? {
-//
-//            let next_precedence = self.get_precedence(&token);
-//
-//            if precedence >= next_precedence {
-//                break;
-//            }
-//
-//            expr = self.parse_infix(expr, next_precedence)?;
-//        }
-//
-//        Ok(expr)
-//    }
-//
 //    fn parse_prefix(&mut self) -> Result<Box<SQLExpr>, ParserError> {
 //
 //        match self.tokenizer.peek_token()? {

src/tokenizer.rs

@@ -50,6 +50,9 @@ pub enum SQLToken<T: Debug + PartialEq> {
 pub trait SQLTokenizer<TokenType>
     where TokenType: Debug + PartialEq {
 
+    /// get the precendence of a token
+    fn precedence(&self, token: &SQLToken<TokenType>) -> usize;
+
     /// return a reference to the next token but do not advance the index
     fn peek_token(&self, chars: &mut Peekable<Chars>) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;