From 69a140a8207c834e86c12227e171b60167be2391 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 2 Sep 2018 18:59:30 -0600 Subject: [PATCH] create Chars struct instead of using Peekable --- examples/acme_parser.rs | 52 +++++++++++++++++------------------------ src/ansi/parser.rs | 2 +- src/ansi/tokenizer.rs | 13 +---------- src/parser.rs | 6 ++--- src/tokenizer.rs | 50 ++++++++++++++++++++++++++++++--------- 5 files changed, 64 insertions(+), 59 deletions(-) diff --git a/examples/acme_parser.rs b/examples/acme_parser.rs index 152215fa..a3fe843a 100644 --- a/examples/acme_parser.rs +++ b/examples/acme_parser.rs @@ -35,40 +35,30 @@ impl SQLTokenizer for AcmeTokenizer { unimplemented!() } - fn peek_token(&mut self) -> Result>, TokenizerError> { + fn next_token(&mut self) -> Result>, TokenizerError> { +// let mut arc = self.ansi_tokenizer.lock().unwrap(); +// match arc.peek_char() { +// Some(&ch) => match ch { +// '!' => { +// arc.next_char(); // consume the first `!` +// match arc.peek_char() { +// Some(&ch) => match ch { +// '!' => { +// arc.next_char(); // consume the second `!` +// Ok(Some(SQLToken::Custom(AcmeToken::Factorial))) +// }, +// _ => Err(TokenizerError::UnexpectedChar(ch,Position::new(0,0))) +// }, +// None => Ok(Some(SQLToken::Not)) +// } +// } +// _ => arc.next_token() +// } +// _ => arc.next_token() +// } unimplemented!() } - fn next_token(&mut self) -> Result>, TokenizerError> { - let mut arc = self.ansi_tokenizer.lock().unwrap(); - match arc.peek_char() { - Some(&ch) => match ch { - '!' => { - arc.next_char(); // consume the first `!` - match arc.peek_char() { - Some(&ch) => match ch { - '!' => { - arc.next_char(); // consume the second `!` - Ok(Some(SQLToken::Custom(AcmeToken::Factorial))) - }, - _ => Err(TokenizerError::UnexpectedChar(ch,Position::new(0,0))) - }, - None => Ok(Some(SQLToken::Not)) - } - } - _ => arc.next_token() - } - _ => arc.next_token() - } - } - - fn peek_char(&mut self) -> Option<&char> { - unimplemented!() - } - - fn next_char(&mut self) -> Option<&char> { - unimplemented!() - } } struct AcmeParser { diff --git a/src/ansi/parser.rs b/src/ansi/parser.rs index 8ed91029..2857cb99 100644 --- a/src/ansi/parser.rs +++ b/src/ansi/parser.rs @@ -24,7 +24,7 @@ impl SQLParser for ANSISQLParser Result>, ParserError> { - match self.tokenizer.lock().unwrap().peek_token()? { + match self.tokenizer.lock().unwrap().next_token()? { Some(SQLToken::Keyword(ref k)) => match k.to_uppercase().as_ref() { "INSERT" => unimplemented!(), "UPDATE" => unimplemented!(), diff --git a/src/ansi/tokenizer.rs b/src/ansi/tokenizer.rs index ef744f27..f61e9b76 100644 --- a/src/ansi/tokenizer.rs +++ b/src/ansi/tokenizer.rs @@ -16,11 +16,7 @@ impl<'a, TokenType> SQLTokenizer for ANSISQLTokenizer<'a> unimplemented!() } - fn peek_token(&mut self) -> Result>, TokenizerError> { - unimplemented!() - } - - fn next_token(&mut self) -> Result>, TokenizerError> { + fn next_token(&mut self) -> Result>, TokenizerError> { match self.chars.next() { Some(ch) => match ch { ' ' | '\t' | '\n' => Ok(Some(SQLToken::Whitespace(ch))), @@ -48,12 +44,5 @@ impl<'a, TokenType> SQLTokenizer for ANSISQLTokenizer<'a> } } - fn peek_char(&mut self) -> Option<&char> { - unimplemented!() - } - - fn next_char(&mut self) -> Option<&char> { - unimplemented!() - } } diff --git a/src/parser.rs b/src/parser.rs index a310ebed..f514e38d 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,8 +1,6 @@ use std::cmp::PartialEq; use std::fmt::Debug; use std::rc::Rc; -use std::str::Chars; -use std::iter::Peekable; use std::sync::{Arc, Mutex}; use super::tokenizer::*; @@ -99,10 +97,10 @@ pub enum ParserError Custom(String) } -impl From> for ParserError +impl From for ParserError where TokenType: Debug + PartialEq { - fn from(_: TokenizerError) -> Self { + fn from(_: TokenizerError) -> Self { unimplemented!() } } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6db89375..9028456a 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -3,6 +3,41 @@ use std::fmt::Debug; //use std::iter::Peekable; //use std::str::Chars; + +pub struct CharSeq { + chars: Vec, + i: usize, + m: usize +} + +impl CharSeq { + + pub fn new(sql: &str) -> Self { + CharSeq { + chars: sql.chars().collect(), + i: 0, + m: 0 + } + } + + pub fn mark(&mut self) { + self.m = self.i; + } + + pub fn reset(&mut self) { + self.i = self.m; + } + + pub fn next(&mut self) -> Option { + if self.i < self.chars.len() { + self.i += 1; + Some(self.chars[self.i-1]) + } else { + None + } + } +} + #[derive(Debug)] pub struct Position { line: usize, @@ -15,11 +50,11 @@ impl Position { } #[derive(Debug)] -pub enum TokenizerError { +pub enum TokenizerError { UnexpectedChar(char,Position), UnexpectedEof(Position), UnterminatedStringLiteral(Position), - Custom(T) + Custom(String) } /// SQL Tokens @@ -43,7 +78,7 @@ pub enum SQLToken { LParen, RParen, Comma, - /// Custom token + /// Custom token (dialect-specific) Custom(T) } @@ -53,15 +88,8 @@ pub trait SQLTokenizer /// get the precendence of a token fn precedence(&self, token: &SQLToken) -> usize; - /// return a reference to the next token but do not advance the index - fn peek_token(&mut self) -> Result>, TokenizerError>; - /// return a reference to the next token and advance the index - fn next_token(&mut self) -> Result>, TokenizerError>; - - fn peek_char(&mut self) -> Option<&char>; - - fn next_char(&mut self) -> Option<&char>; + fn next_token(&mut self) -> Result>, TokenizerError>; } //