mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-04 06:18:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

tokenizer delegates to dialect now

Browse source
This commit is contained in:
Andy Grove 2018-09-08 14:49:25 -06:00
parent 96f1f9f35e
commit 810cd8e6cf
4 changed files with 449 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

4
examples/parse_select.rs
View file

@ -9,9 +9,9 @@ fn main() {
WHERE a > b AND b < 100 \ WHERE a > b AND b < 100 \
ORDER BY a DESC, b"; ORDER BY a DESC, b";
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let ast = Parser::parse_sql(&dialect,sql.to_string()).unwrap(); let ast = Parser::parse_sql(&dialect, sql.to_string()).unwrap();
println!("AST: {:?}", ast); println!("AST: {:?}", ast);
} }

516
src/dialect.rs
View file

@ -17,54 +17,353 @@
pub trait Dialect { pub trait Dialect {
/// Get a list of keywords for this dialect /// Get a list of keywords for this dialect
fn keywords(&self) -> Vec<&'static str>; fn keywords(&self) -> Vec<&'static str>;
/// Determine if a character is a valid identifier start character
fn is_identifier_start(&self, ch: char) -> bool;
/// Determine if a character is a valid identifier character
fn is_identifier_part(&self, ch: char) -> bool;
} }
pub struct AnsiSqlDialect { pub struct AnsiSqlDialect {}
}
impl Dialect for AnsiSqlDialect { impl Dialect for AnsiSqlDialect {
fn keywords(&self) -> Vec<&'static str> { fn keywords(&self) -> Vec<&'static str> {
return vec!["ABS", "ALL", "ALLOCATE", "ALTER", "AND", "ANY", "ARE", "ARRAY", "ARRAY_AGG", return vec![
"ARRAY_MAX_CARDINALITY", "AS", "ASENSITIVE", "ASYMMETRIC", "AT", "ATOMIC", "AUTHORIZATION", "ABS",
"AVG", "BEGIN", "BEGIN_FRAME", "BEGIN_PARTITION", "BETWEEN", "BIGINT", "BINARY", "BLOB", "ALL",
"BOOLEAN", "BOTH", "BY", "CALL", "CALLED", "CARDINALITY", "CASCADED", "CASE", "CAST", "CEIL", "ALLOCATE",
"CEILING", "CHAR", "CHAR_LENGTH", "CHARACTER", "CHARACTER_LENGTH", "CHECK", "CLOB", "CLOSE", "ALTER",
"COALESCE", "COLLATE", "COLLECT", "COLUMN", "COMMIT", "CONDITION", "CONNECT", "CONSTRAINT", "AND",
"CONTAINS", "CONVERT", "CORR", "CORRESPONDING", "COUNT", "COVAR_POP", "COVAR_SAMP", "CREATE", "ANY",
"CROSS", "CUBE", "CUME_DIST", "CURRENT", "CURRENT_CATALOG", "CURRENT_DATE", "ARE",
"CURRENT_DEFAULT_TRANSFORM_GROUP", "CURRENT_PATH", "CURRENT_ROLE", "CURRENT_ROW", "ARRAY",
"CURRENT_SCHEMA", "CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_TRANSFORM_GROUP_FOR_TYPE", "ARRAY_AGG",
"CURRENT_USER", "CURSOR", "CYCLE", "DATE", "DAY", "DEALLOCATE", "DEC", "DECIMAL", "DECLARE", "ARRAY_MAX_CARDINALITY",
"DEFAULT", "DELETE", "DENSE_RANK", "DEREF", "DESCRIBE", "DETERMINISTIC", "DISCONNECT", "AS",
"DISTINCT", "DOUBLE", "DROP", "DYNAMIC", "EACH", "ELEMENT", "ELSE", "END", "END_FRAME", "ASENSITIVE",
"END_PARTITION", "END-EXEC", "EQUALS", "ESCAPE", "EVERY", "EXCEPT", "EXEC", "EXECUTE", "ASYMMETRIC",
"EXISTS", "EXP", "EXTERNAL", "EXTRACT", "FALSE", "FETCH", "FILTER", "FIRST_VALUE", "FLOAT", "AT",
"FLOOR", "FOR", "FOREIGN", "FRAME_ROW", "FREE", "FROM", "FULL", "FUNCTION", "FUSION", "ATOMIC",
"GET", "GLOBAL", "GRANT", "GROUP", "GROUPING", "GROUPS", "HAVING", "HOLD", "HOUR", "IDENTITY", "AUTHORIZATION",
"IN", "INDICATOR", "INNER", "INOUT", "INSENSITIVE", "INSERT", "INT", "INTEGER", "INTERSECT", "AVG",
"INTERSECTION", "INTERVAL", "INTO", "IS", "JOIN", "LAG", "LANGUAGE", "LARGE", "LAST_VALUE", "BEGIN",
"LATERAL", "LEAD", "LEADING", "LEFT", "LIKE", "LIKE_REGEX", "LN", "LOCAL", "LOCALTIME", "BEGIN_FRAME",
"LOCALTIMESTAMP", "LOWER", "MATCH", "MAX", "MEMBER", "MERGE", "METHOD", "MIN", "MINUTE", "BEGIN_PARTITION",
"MOD", "MODIFIES", "MODULE", "MONTH", "MULTISET", "NATIONAL", "NATURAL", "NCHAR", "NCLOB", "BETWEEN",
"NEW", "NO", "NONE", "NORMALIZE", "NOT", "NTH_VALUE", "NTILE", "NULL", "NULLIF", "NUMERIC", "BIGINT",
"OCTET_LENGTH", "OCCURRENCES_REGEX", "OF", "OFFSET", "OLD", "ON", "ONLY", "OPEN", "OR", "BINARY",
"ORDER", "OUT", "OUTER", "OVER", "OVERLAPS", "OVERLAY", "PARAMETER", "PARTITION", "PERCENT", "BLOB",
"PERCENT_RANK", "PERCENTILE_CONT", "PERCENTILE_DISC", "PERIOD", "PORTION", "POSITION", "BOOLEAN",
"POSITION_REGEX", "POWER", "PRECEDES", "PRECISION", "PREPARE", "PRIMARY", "BOTH",
"PROCEDURE", "RANGE", "RANK", "READS", "REAL", "RECURSIVE", "REF", "REFERENCES", "BY",
"REFERENCING", "REGR_AVGX", "REGR_AVGY", "REGR_COUNT", "REGR_INTERCEPT", "REGR_R2", "CALL",
"REGR_SLOPE", "REGR_SXX", "REGR_SXY", "REGR_SYY", "RELEASE", "RESULT", "RETURN", "RETURNS", "CALLED",
"REVOKE", "RIGHT", "ROLLBACK", "ROLLUP", "ROW", "ROW_NUMBER", "ROWS", "SAVEPOINT", "CARDINALITY",
"SCOPE", "SCROLL", "SEARCH", "SECOND", "SELECT", "SENSITIVE", "SESSION_USER", "SET", "CASCADED",
"SIMILAR", "SMALLINT", "SOME", "SPECIFIC", "SPECIFICTYPE", "SQL", "SQLEXCEPTION", "SQLSTATE", "CASE",
"SQLWARNING", "SQRT", "START", "STATIC", "STDDEV_POP", "STDDEV_SAMP", "SUBMULTISET", "CAST",
"SUBSTRING", "SUBSTRING_REGEX", "SUCCEEDS", "SUM", "SYMMETRIC", "SYSTEM", "SYSTEM_TIME", "CEIL",
"SYSTEM_USER", "TABLE", "TABLESAMPLE", "THEN", "TIME", "TIMESTAMP", "TIMEZONE_HOUR", "CEILING",
"TIMEZONE_MINUTE", "TO", "TRAILING", "TRANSLATE", "TRANSLATE_REGEX", "TRANSLATION", "CHAR",
"TREAT", "TRIGGER", "TRUNCATE", "TRIM", "TRIM_ARRAY", "TRUE", "UESCAPE", "UNION", "UNIQUE", "CHAR_LENGTH",
"UNKNOWN", "UNNEST", "UPDATE", "UPPER", "USER", "USING", "VALUE", "VALUES", "VALUE_OF", "CHARACTER",
"VAR_POP", "VAR_SAMP", "VARBINARY", "VARCHAR", "VARYING", "VERSIONING", "WHEN", "WHENEVER", "CHARACTER_LENGTH",
"WHERE", "WIDTH_BUCKET", "WINDOW", "WITH", "WITHIN", "WITHOUT", "YEAR"]; "CHECK",
"CLOB",
"CLOSE",
"COALESCE",
"COLLATE",
"COLLECT",
"COLUMN",
"COMMIT",
"CONDITION",
"CONNECT",
"CONSTRAINT",
"CONTAINS",
"CONVERT",
"CORR",
"CORRESPONDING",
"COUNT",
"COVAR_POP",
"COVAR_SAMP",
"CREATE",
"CROSS",
"CUBE",
"CUME_DIST",
"CURRENT",
"CURRENT_CATALOG",
"CURRENT_DATE",
"CURRENT_DEFAULT_TRANSFORM_GROUP",
"CURRENT_PATH",
"CURRENT_ROLE",
"CURRENT_ROW",
"CURRENT_SCHEMA",
"CURRENT_TIME",
"CURRENT_TIMESTAMP",
"CURRENT_TRANSFORM_GROUP_FOR_TYPE",
"CURRENT_USER",
"CURSOR",
"CYCLE",
"DATE",
"DAY",
"DEALLOCATE",
"DEC",
"DECIMAL",
"DECLARE",
"DEFAULT",
"DELETE",
"DENSE_RANK",
"DEREF",
"DESCRIBE",
"DETERMINISTIC",
"DISCONNECT",
"DISTINCT",
"DOUBLE",
"DROP",
"DYNAMIC",
"EACH",
"ELEMENT",
"ELSE",
"END",
"END_FRAME",
"END_PARTITION",
"END-EXEC",
"EQUALS",
"ESCAPE",
"EVERY",
"EXCEPT",
"EXEC",
"EXECUTE",
"EXISTS",
"EXP",
"EXTERNAL",
"EXTRACT",
"FALSE",
"FETCH",
"FILTER",
"FIRST_VALUE",
"FLOAT",
"FLOOR",
"FOR",
"FOREIGN",
"FRAME_ROW",
"FREE",
"FROM",
"FULL",
"FUNCTION",
"FUSION",
"GET",
"GLOBAL",
"GRANT",
"GROUP",
"GROUPING",
"GROUPS",
"HAVING",
"HOLD",
"HOUR",
"IDENTITY",
"IN",
"INDICATOR",
"INNER",
"INOUT",
"INSENSITIVE",
"INSERT",
"INT",
"INTEGER",
"INTERSECT",
"INTERSECTION",
"INTERVAL",
"INTO",
"IS",
"JOIN",
"LAG",
"LANGUAGE",
"LARGE",
"LAST_VALUE",
"LATERAL",
"LEAD",
"LEADING",
"LEFT",
"LIKE",
"LIKE_REGEX",
"LN",
"LOCAL",
"LOCALTIME",
"LOCALTIMESTAMP",
"LOWER",
"MATCH",
"MAX",
"MEMBER",
"MERGE",
"METHOD",
"MIN",
"MINUTE",
"MOD",
"MODIFIES",
"MODULE",
"MONTH",
"MULTISET",
"NATIONAL",
"NATURAL",
"NCHAR",
"NCLOB",
"NEW",
"NO",
"NONE",
"NORMALIZE",
"NOT",
"NTH_VALUE",
"NTILE",
"NULL",
"NULLIF",
"NUMERIC",
"OCTET_LENGTH",
"OCCURRENCES_REGEX",
"OF",
"OFFSET",
"OLD",
"ON",
"ONLY",
"OPEN",
"OR",
"ORDER",
"OUT",
"OUTER",
"OVER",
"OVERLAPS",
"OVERLAY",
"PARAMETER",
"PARTITION",
"PERCENT",
"PERCENT_RANK",
"PERCENTILE_CONT",
"PERCENTILE_DISC",
"PERIOD",
"PORTION",
"POSITION",
"POSITION_REGEX",
"POWER",
"PRECEDES",
"PRECISION",
"PREPARE",
"PRIMARY",
"PROCEDURE",
"RANGE",
"RANK",
"READS",
"REAL",
"RECURSIVE",
"REF",
"REFERENCES",
"REFERENCING",
"REGR_AVGX",
"REGR_AVGY",
"REGR_COUNT",
"REGR_INTERCEPT",
"REGR_R2",
"REGR_SLOPE",
"REGR_SXX",
"REGR_SXY",
"REGR_SYY",
"RELEASE",
"RESULT",
"RETURN",
"RETURNS",
"REVOKE",
"RIGHT",
"ROLLBACK",
"ROLLUP",
"ROW",
"ROW_NUMBER",
"ROWS",
"SAVEPOINT",
"SCOPE",
"SCROLL",
"SEARCH",
"SECOND",
"SELECT",
"SENSITIVE",
"SESSION_USER",
"SET",
"SIMILAR",
"SMALLINT",
"SOME",
"SPECIFIC",
"SPECIFICTYPE",
"SQL",
"SQLEXCEPTION",
"SQLSTATE",
"SQLWARNING",
"SQRT",
"START",
"STATIC",
"STDDEV_POP",
"STDDEV_SAMP",
"SUBMULTISET",
"SUBSTRING",
"SUBSTRING_REGEX",
"SUCCEEDS",
"SUM",
"SYMMETRIC",
"SYSTEM",
"SYSTEM_TIME",
"SYSTEM_USER",
"TABLE",
"TABLESAMPLE",
"THEN",
"TIME",
"TIMESTAMP",
"TIMEZONE_HOUR",
"TIMEZONE_MINUTE",
"TO",
"TRAILING",
"TRANSLATE",
"TRANSLATE_REGEX",
"TRANSLATION",
"TREAT",
"TRIGGER",
"TRUNCATE",
"TRIM",
"TRIM_ARRAY",
"TRUE",
"UESCAPE",
"UNION",
"UNIQUE",
"UNKNOWN",
"UNNEST",
"UPDATE",
"UPPER",
"USER",
"USING",
"VALUE",
"VALUES",
"VALUE_OF",
"VAR_POP",
"VAR_SAMP",
"VARBINARY",
"VARCHAR",
"VARYING",
"VERSIONING",
"WHEN",
"WHENEVER",
"WHERE",
"WIDTH_BUCKET",
"WINDOW",
"WITH",
"WITHIN",
"WITHOUT",
"YEAR",
];
}
fn is_identifier_start(&self, ch: char) -> bool {
(ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z')
}
fn is_identifier_part(&self, ch: char) -> bool {
(ch >= 'a' && ch <= 'z')
|| (ch >= 'A' && ch <= 'Z')
|| (ch >= '0' && ch <= '9')
|| ch == '_'
} }
} }
@ -75,67 +374,76 @@ impl Dialect for GenericSqlDialect {
return vec![ return vec![
"SELECT", "SELECT",
"FROM", "FROM",
"WHERE", "WHERE",
"LIMIT", "LIMIT",
"ORDER", "ORDER",
"GROUP", "GROUP",
"BY", "BY",
"HAVING", "HAVING",
"UNION", "UNION",
"ALL", "ALL",
"INSERT", "INSERT",
"UPDATE", "UPDATE",
"DELETE", "DELETE",
"IN", "IN",
"IS", "IS",
"NULL", "NULL",
"SET", "SET",
"CREATE", "CREATE",
"EXTERNAL", "EXTERNAL",
"TABLE", "TABLE",
"ASC", "ASC",
"DESC", "DESC",
"AND", "AND",
"OR", "OR",
"NOT", "NOT",
"AS", "AS",
"STORED", "STORED",
"CSV", "CSV",
"PARQUET", "PARQUET",
"LOCATION", "LOCATION",
"WITH", "WITH",
"WITHOUT", "WITHOUT",
"HEADER", "HEADER",
"ROW", "ROW",
// SQL types
// SQL types "CHAR",
"CHAR", "CHARACTER",
"CHARACTER", "VARYING",
"VARYING", "LARGE",
"LARGE", "OBJECT",
"OBJECT", "VARCHAR",
"VARCHAR", "CLOB",
"CLOB", "BINARY",
"BINARY", "VARBINARY",
"VARBINARY", "BLOB",
"BLOB", "FLOAT",
"FLOAT", "REAL",
"REAL", "DOUBLE",
"DOUBLE", "PRECISION",
"PRECISION", "INT",
"INT", "INTEGER",
"INTEGER", "SMALLINT",
"SMALLINT", "BIGINT",
"BIGINT", "NUMERIC",
"NUMERIC", "DECIMAL",
"DECIMAL", "DEC",
"DEC", "BOOLEAN",
"BOOLEAN", "DATE",
"DATE", "TIME",
"TIME", "TIMESTAMP",
"TIMESTAMP",
]; ];
} }
}
fn is_identifier_start(&self, ch: char) -> bool {
(ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '@'
}
fn is_identifier_part(&self, ch: char) -> bool {
(ch >= 'a' && ch <= 'z')
|| (ch >= 'A' && ch <= 'Z')
|| (ch >= '0' && ch <= '9')
|| ch == '@'
|| ch == '_'
}
}

4
src/sqlparser.rs
View file

@ -619,8 +619,8 @@ impl Parser {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*;
use super::super::dialect::GenericSqlDialect; use super::super::dialect::GenericSqlDialect;
use super::*;
#[test] #[test]
fn parse_delete_statement() { fn parse_delete_statement() {
@ -952,7 +952,7 @@ mod tests {
fn parse_sql(sql: &str) -> ASTNode { fn parse_sql(sql: &str) -> ASTNode {
let dialect = GenericSqlDialect {}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql, ); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
let mut parser = Parser::new(tokens); let mut parser = Parser::new(tokens);
let ast = parser.parse().unwrap(); let ast = parser.parse().unwrap();

62
src/sqltokenizer.rs
View file

@ -13,6 +13,10 @@
// limitations under the License. // limitations under the License.
//! SQL Tokenizer //! SQL Tokenizer
//!
//! The tokenizer (a.k.a. lexer) converts a string into a sequence of tokens.
//!
//! The tokens then form the input for the parser, which outputs an Abstract Syntax Tree (AST).
use std::iter::Peekable; use std::iter::Peekable;
use std::str::Chars; use std::str::Chars;
@ -69,18 +73,18 @@ pub enum Token {
pub struct TokenizerError(String); pub struct TokenizerError(String);
/// SQL Tokenizer /// SQL Tokenizer
pub struct Tokenizer { pub struct Tokenizer<'a> {
keywords: Vec<&'static str>, dialect: &'a Dialect,
pub query: String, pub query: String,
pub line: u64, pub line: u64,
pub col: u64, pub col: u64,
} }
impl Tokenizer { impl<'a> Tokenizer<'a> {
/// Create a new SQL tokenizer for the specified SQL statement /// Create a new SQL tokenizer for the specified SQL statement
pub fn new(dialect: &Dialect, query: &str) -> Self { pub fn new(dialect: &'a Dialect, query: &str) -> Self {
Self { Self {
keywords: dialect.keywords(), dialect,
query: query.to_string(), query: query.to_string(),
line: 1, line: 1,
col: 1, col: 1,
@ -91,8 +95,7 @@ impl Tokenizer {
//TODO: need to reintroduce FnvHashSet at some point .. iterating over keywords is //TODO: need to reintroduce FnvHashSet at some point .. iterating over keywords is
// not fast but I want the simplicity for now while I experiment with pluggable // not fast but I want the simplicity for now while I experiment with pluggable
// dialects // dialects
return self.keywords.contains(&s); return self.dialect.keywords().contains(&s);
} }
/// Tokenize the statement and produce a vector of tokens /// Tokenize the statement and produce a vector of tokens
@ -138,15 +141,16 @@ impl Tokenizer {
Ok(Some(Token::Whitespace(ch))) Ok(Some(Token::Whitespace(ch)))
} }
// identifier or keyword // identifier or keyword
'a'...'z' | 'A'...'Z' | '_' | '@' => { ch if self.dialect.is_identifier_start(ch) => {
let mut s = String::new(); let mut s = String::new();
chars.next(); // consume
s.push(ch);
while let Some(&ch) = chars.peek() { while let Some(&ch) = chars.peek() {
match ch { if self.dialect.is_identifier_part(ch) {
'a'...'z' | 'A'...'Z' | '_' | '0'...'9' | '@' => { chars.next(); // consume
chars.next(); // consume s.push(ch);
s.push(ch); } else {
} break;
_ => break,
} }
} }
let upper_str = s.to_uppercase(); let upper_str = s.to_uppercase();
@ -293,14 +297,14 @@ impl Tokenizer {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::super::dialect::GenericSqlDialect;
use super::*; use super::*;
use super::super::dialect::{GenericSqlDialect};
#[test] #[test]
fn tokenize_select_1() { fn tokenize_select_1() {
let sql = String::from("SELECT 1"); let sql = String::from("SELECT 1");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
let expected = vec![ let expected = vec![
@ -314,8 +318,8 @@ mod tests {
#[test] #[test]
fn tokenize_scalar_function() { fn tokenize_scalar_function() {
let sql = String::from("SELECT sqrt(1)"); let sql = String::from("SELECT sqrt(1)");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
let expected = vec![ let expected = vec![
@ -332,8 +336,8 @@ mod tests {
#[test] #[test]
fn tokenize_simple_select() { fn tokenize_simple_select() {
let sql = String::from("SELECT * FROM customer WHERE id = 1 LIMIT 5"); let sql = String::from("SELECT * FROM customer WHERE id = 1 LIMIT 5");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
let expected = vec![ let expected = vec![
@ -355,8 +359,8 @@ mod tests {
#[test] #[test]
fn tokenize_string_predicate() { fn tokenize_string_predicate() {
let sql = String::from("SELECT * FROM customer WHERE salary != 'Not Provided'"); let sql = String::from("SELECT * FROM customer WHERE salary != 'Not Provided'");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
let expected = vec![ let expected = vec![
@ -377,8 +381,8 @@ mod tests {
fn tokenize_invalid_string() { fn tokenize_invalid_string() {
let sql = String::from("\nمصطفىh"); let sql = String::from("\nمصطفىh");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize(); let tokens = tokenizer.tokenize();
match tokens { match tokens {
@ -396,8 +400,8 @@ mod tests {
fn tokenize_invalid_string_cols() { fn tokenize_invalid_string_cols() {
let sql = String::from("\n\nSELECT * FROM table\tمصطفىh"); let sql = String::from("\n\nSELECT * FROM table\tمصطفىh");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize(); let tokens = tokenizer.tokenize();
match tokens { match tokens {
Err(e) => assert_eq!( Err(e) => assert_eq!(
@ -413,8 +417,8 @@ mod tests {
#[test] #[test]
fn tokenize_is_null() { fn tokenize_is_null() {
let sql = String::from("a IS NULL"); let sql = String::from("a IS NULL");
let dialect = GenericSqlDialect{}; let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect,&sql); let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap(); let tokens = tokenizer.tokenize().unwrap();
let expected = vec![ let expected = vec![