create Chars struct instead of using Peekable

src/tokenizer.rs

@@ -3,6 +3,41 @@ use std::fmt::Debug;
 //use std::iter::Peekable;
 //use std::str::Chars;
 
+
+pub struct CharSeq {
+    chars: Vec<char>,
+    i: usize,
+    m: usize
+}
+
+impl CharSeq {
+
+    pub fn new(sql: &str) -> Self {
+        CharSeq {
+            chars: sql.chars().collect(),
+            i: 0,
+            m: 0
+        }
+    }
+
+    pub fn mark(&mut self) {
+        self.m = self.i;
+    }
+
+    pub fn reset(&mut self) {
+        self.i = self.m;
+    }
+
+    pub fn next(&mut self) -> Option<char> {
+        if self.i < self.chars.len() {
+            self.i += 1;
+            Some(self.chars[self.i-1])
+        } else {
+            None
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct Position {
     line: usize,
@@ -15,11 +50,11 @@ impl Position {
 }
 
 #[derive(Debug)]
-pub enum TokenizerError<T> {
+pub enum TokenizerError {
     UnexpectedChar(char,Position),
     UnexpectedEof(Position),
     UnterminatedStringLiteral(Position),
-    Custom(T)
+    Custom(String)
 }
 
 /// SQL Tokens
@@ -43,7 +78,7 @@ pub enum SQLToken<T: Debug + PartialEq> {
     LParen,
     RParen,
     Comma,
-    /// Custom token
+    /// Custom token (dialect-specific)
     Custom(T)
 }
 
@@ -53,15 +88,8 @@ pub trait SQLTokenizer<TokenType>
     /// get the precendence of a token
     fn precedence(&self, token: &SQLToken<TokenType>) -> usize;
 
-    /// return a reference to the next token but do not advance the index
-    fn peek_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
-
     /// return a reference to the next token and advance the index
-    fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError<TokenType>>;
-
-    fn peek_char(&mut self) -> Option<&char>;
-
-    fn next_char(&mut self) -> Option<&char>;
+    fn next_token(&mut self) -> Result<Option<SQLToken<TokenType>>, TokenizerError>;
 }
 
 //