mirror of
https://github.com/apache/datafusion-sqlparser-rs.git
synced 2025-10-17 01:07:19 +00:00
feat: mysql no-escape mode (#870)
Co-authored-by: Andrew Lamb <andrew@nerdnetworks.org>
This commit is contained in:
canalun
GitHub
parent
eb288487a6
commit
f98a2f9dca
7 changed files with 485 additions and 121 deletions
245
src/tokenizer.rs
245
src/tokenizer.rs
|
|
@ -455,12 +455,69 @@ impl<'a> State<'a> {
|
|||
pub struct Tokenizer<'a> {
|
||||
dialect: &'a dyn Dialect,
|
||||
query: &'a str,
|
||||
/// If true (the default), the tokenizer will un-escape literal
|
||||
/// SQL strings See [`Tokenizer::with_unescape`] for more details.
|
||||
unescape: bool,
|
||||
}
|
||||
|
||||
impl<'a> Tokenizer<'a> {
|
||||
/// Create a new SQL tokenizer for the specified SQL statement
|
||||
///
|
||||
/// ```
|
||||
/// # use sqlparser::tokenizer::{Token, Whitespace, Tokenizer};
|
||||
/// # use sqlparser::dialect::GenericDialect;
|
||||
/// # let dialect = GenericDialect{};
|
||||
/// let query = r#"SELECT 'foo'"#;
|
||||
///
|
||||
/// // Parsing the query
|
||||
/// let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
|
||||
///
|
||||
/// assert_eq!(tokens, vec![
|
||||
/// Token::make_word("SELECT", None),
|
||||
/// Token::Whitespace(Whitespace::Space),
|
||||
/// Token::SingleQuotedString("foo".to_string()),
|
||||
/// ]);
|
||||
pub fn new(dialect: &'a dyn Dialect, query: &'a str) -> Self {
|
||||
Self { dialect, query }
|
||||
Self {
|
||||
dialect,
|
||||
query,
|
||||
unescape: true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set unescape mode
|
||||
///
|
||||
/// When true (default) the tokenizer unescapes literal values
|
||||
/// (for example, `""` in SQL is unescaped to the literal `"`).
|
||||
///
|
||||
/// When false, the tokenizer provides the raw strings as provided
|
||||
/// in the query. This can be helpful for programs that wish to
|
||||
/// recover the *exact* original query text without normalizing
|
||||
/// the escaping
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// # use sqlparser::tokenizer::{Token, Tokenizer};
|
||||
/// # use sqlparser::dialect::GenericDialect;
|
||||
/// # let dialect = GenericDialect{};
|
||||
/// let query = r#""Foo "" Bar""#;
|
||||
/// let unescaped = Token::make_word(r#"Foo " Bar"#, Some('"'));
|
||||
/// let original = Token::make_word(r#"Foo "" Bar"#, Some('"'));
|
||||
///
|
||||
/// // Parsing with unescaping (default)
|
||||
/// let tokens = Tokenizer::new(&dialect, &query).tokenize().unwrap();
|
||||
/// assert_eq!(tokens, vec![unescaped]);
|
||||
///
|
||||
/// // Parsing with unescape = false
|
||||
/// let tokens = Tokenizer::new(&dialect, &query)
|
||||
/// .with_unescape(false)
|
||||
/// .tokenize().unwrap();
|
||||
/// assert_eq!(tokens, vec![original]);
|
||||
/// ```
|
||||
pub fn with_unescape(mut self, unescape: bool) -> Self {
|
||||
self.unescape = unescape;
|
||||
self
|
||||
}
|
||||
|
||||
/// Tokenize the statement and produce a vector of tokens
|
||||
|
|
@ -650,7 +707,7 @@ impl<'a> Tokenizer<'a> {
|
|||
let error_loc = chars.location();
|
||||
chars.next(); // consume the opening quote
|
||||
let quote_end = Word::matching_end_quote(quote_start);
|
||||
let (s, last_char) = parse_quoted_ident(chars, quote_end);
|
||||
let (s, last_char) = self.parse_quoted_ident(chars, quote_end);
|
||||
|
||||
if last_char == Some(quote_end) {
|
||||
Ok(Some(Token::make_word(&s, Some(quote_start))))
|
||||
|
|
@ -1168,6 +1225,10 @@ impl<'a> Tokenizer<'a> {
|
|||
chars.next(); // consume
|
||||
if chars.peek().map(|c| *c == quote_style).unwrap_or(false) {
|
||||
s.push(ch);
|
||||
if !self.unescape {
|
||||
// In no-escape mode, the given query has to be saved completely
|
||||
s.push(ch);
|
||||
}
|
||||
chars.next();
|
||||
} else {
|
||||
return Ok(s);
|
||||
|
|
@ -1176,22 +1237,29 @@ impl<'a> Tokenizer<'a> {
|
|||
'\\' => {
|
||||
// consume
|
||||
chars.next();
|
||||
// slash escaping is specific to MySQL dialect
|
||||
// slash escaping is specific to MySQL dialect.
|
||||
if dialect_of!(self is MySqlDialect) {
|
||||
if let Some(next) = chars.peek() {
|
||||
// See https://dev.mysql.com/doc/refman/8.0/en/string-literals.html#character-escape-sequences
|
||||
let n = match next {
|
||||
'\'' | '\"' | '\\' | '%' | '_' => *next,
|
||||
'0' => '\0',
|
||||
'b' => '\u{8}',
|
||||
'n' => '\n',
|
||||
'r' => '\r',
|
||||
't' => '\t',
|
||||
'Z' => '\u{1a}',
|
||||
_ => *next,
|
||||
};
|
||||
s.push(n);
|
||||
chars.next(); // consume next
|
||||
if !self.unescape {
|
||||
// In no-escape mode, the given query has to be saved completely including backslashes.
|
||||
s.push(ch);
|
||||
s.push(*next);
|
||||
chars.next(); // consume next
|
||||
} else {
|
||||
// See https://dev.mysql.com/doc/refman/8.0/en/string-literals.html#character-escape-sequences
|
||||
let n = match next {
|
||||
'\'' | '\"' | '\\' | '%' | '_' => *next,
|
||||
'0' => '\0',
|
||||
'b' => '\u{8}',
|
||||
'n' => '\n',
|
||||
'r' => '\r',
|
||||
't' => '\t',
|
||||
'Z' => '\u{1a}',
|
||||
_ => *next,
|
||||
};
|
||||
s.push(n);
|
||||
chars.next(); // consume next
|
||||
}
|
||||
}
|
||||
} else {
|
||||
s.push(ch);
|
||||
|
|
@ -1239,6 +1307,29 @@ impl<'a> Tokenizer<'a> {
|
|||
}
|
||||
}
|
||||
|
||||
fn parse_quoted_ident(&self, chars: &mut State, quote_end: char) -> (String, Option<char>) {
|
||||
let mut last_char = None;
|
||||
let mut s = String::new();
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch == quote_end {
|
||||
if chars.peek() == Some("e_end) {
|
||||
chars.next();
|
||||
s.push(ch);
|
||||
if !self.unescape {
|
||||
// In no-escape mode, the given query has to be saved completely
|
||||
s.push(ch);
|
||||
}
|
||||
} else {
|
||||
last_char = Some(quote_end);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
s.push(ch);
|
||||
}
|
||||
}
|
||||
(s, last_char)
|
||||
}
|
||||
|
||||
#[allow(clippy::unnecessary_wraps)]
|
||||
fn consume_and_return(
|
||||
&self,
|
||||
|
|
@ -1266,25 +1357,6 @@ fn peeking_take_while(chars: &mut State, mut predicate: impl FnMut(char) -> bool
|
|||
s
|
||||
}
|
||||
|
||||
fn parse_quoted_ident(chars: &mut State, quote_end: char) -> (String, Option<char>) {
|
||||
let mut last_char = None;
|
||||
let mut s = String::new();
|
||||
while let Some(ch) = chars.next() {
|
||||
if ch == quote_end {
|
||||
if chars.peek() == Some("e_end) {
|
||||
chars.next();
|
||||
s.push(ch);
|
||||
} else {
|
||||
last_char = Some(quote_end);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
s.push(ch);
|
||||
}
|
||||
}
|
||||
(s, last_char)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -1309,8 +1381,7 @@ mod tests {
|
|||
fn tokenize_select_1() {
|
||||
let sql = String::from("SELECT 1");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1325,8 +1396,7 @@ mod tests {
|
|||
fn tokenize_select_float() {
|
||||
let sql = String::from("SELECT .1");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1341,8 +1411,7 @@ mod tests {
|
|||
fn tokenize_select_exponent() {
|
||||
let sql = String::from("SELECT 1e10, 1e-10, 1e+10, 1ea, 1e-10a, 1e-10-10");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1376,8 +1445,7 @@ mod tests {
|
|||
fn tokenize_scalar_function() {
|
||||
let sql = String::from("SELECT sqrt(1)");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1395,8 +1463,7 @@ mod tests {
|
|||
fn tokenize_string_string_concat() {
|
||||
let sql = String::from("SELECT 'a' || 'b'");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1414,8 +1481,7 @@ mod tests {
|
|||
fn tokenize_bitwise_op() {
|
||||
let sql = String::from("SELECT one | two ^ three");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1438,8 +1504,7 @@ mod tests {
|
|||
let sql =
|
||||
String::from("SELECT true XOR true, false XOR false, true XOR false, false XOR true");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1478,8 +1543,7 @@ mod tests {
|
|||
fn tokenize_simple_select() {
|
||||
let sql = String::from("SELECT * FROM customer WHERE id = 1 LIMIT 5");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1510,8 +1574,7 @@ mod tests {
|
|||
fn tokenize_explain_select() {
|
||||
let sql = String::from("EXPLAIN SELECT * FROM customer WHERE id = 1");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("EXPLAIN"),
|
||||
|
|
@ -1540,8 +1603,7 @@ mod tests {
|
|||
fn tokenize_explain_analyze_select() {
|
||||
let sql = String::from("EXPLAIN ANALYZE SELECT * FROM customer WHERE id = 1");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("EXPLAIN"),
|
||||
|
|
@ -1572,8 +1634,7 @@ mod tests {
|
|||
fn tokenize_string_predicate() {
|
||||
let sql = String::from("SELECT * FROM customer WHERE salary != 'Not Provided'");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
|
|
@ -1601,8 +1662,7 @@ mod tests {
|
|||
let sql = String::from("\n💝مصطفىh");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
// println!("tokens: {:#?}", tokens);
|
||||
let expected = vec![
|
||||
Token::Whitespace(Whitespace::Newline),
|
||||
|
|
@ -1617,8 +1677,7 @@ mod tests {
|
|||
let sql = String::from("'foo\r\nbar\nbaz'");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![Token::SingleQuotedString("foo\r\nbar\nbaz".to_string())];
|
||||
compare(expected, tokens);
|
||||
}
|
||||
|
|
@ -1660,8 +1719,7 @@ mod tests {
|
|||
let sql = String::from("\n\nSELECT * FROM table\t💝مصطفىh");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
// println!("tokens: {:#?}", tokens);
|
||||
let expected = vec![
|
||||
Token::Whitespace(Whitespace::Newline),
|
||||
|
|
@ -1684,8 +1742,7 @@ mod tests {
|
|||
fn tokenize_right_arrow() {
|
||||
let sql = String::from("FUNCTION(key=>value)");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::make_word("FUNCTION", None),
|
||||
Token::LParen,
|
||||
|
|
@ -1701,8 +1758,7 @@ mod tests {
|
|||
fn tokenize_is_null() {
|
||||
let sql = String::from("a IS NULL");
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
|
||||
let expected = vec![
|
||||
Token::make_word("a", None),
|
||||
|
|
@ -1720,8 +1776,7 @@ mod tests {
|
|||
let sql = String::from("0--this is a comment\n1");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::Number("0".to_string(), false),
|
||||
Token::Whitespace(Whitespace::SingleLineComment {
|
||||
|
|
@ -1738,8 +1793,7 @@ mod tests {
|
|||
let sql = String::from("--this is a comment");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![Token::Whitespace(Whitespace::SingleLineComment {
|
||||
prefix: "--".to_string(),
|
||||
comment: "this is a comment".to_string(),
|
||||
|
|
@ -1752,8 +1806,7 @@ mod tests {
|
|||
let sql = String::from("0/*multi-line\n* /comment*/1");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::Number("0".to_string(), false),
|
||||
Token::Whitespace(Whitespace::MultiLineComment(
|
||||
|
|
@ -1769,8 +1822,7 @@ mod tests {
|
|||
let sql = String::from("0/*multi-line\n* \n/* comment \n /*comment*/*/ */ /comment*/1");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::Number("0".to_string(), false),
|
||||
Token::Whitespace(Whitespace::MultiLineComment(
|
||||
|
|
@ -1786,8 +1838,7 @@ mod tests {
|
|||
let sql = String::from("\n/** Comment **/\n");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::Whitespace(Whitespace::Newline),
|
||||
Token::Whitespace(Whitespace::MultiLineComment("* Comment *".to_string())),
|
||||
|
|
@ -1801,8 +1852,7 @@ mod tests {
|
|||
let sql = String::from(" \u{2003}\n");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
|
|
@ -1832,8 +1882,7 @@ mod tests {
|
|||
let sql = String::from("line1\nline2\rline3\r\nline4\r");
|
||||
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, &sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::make_word("line1", None),
|
||||
Token::Whitespace(Whitespace::Newline),
|
||||
|
|
@ -1851,8 +1900,7 @@ mod tests {
|
|||
fn tokenize_mssql_top() {
|
||||
let sql = "SELECT TOP 5 [bar] FROM foo";
|
||||
let dialect = MsSqlDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
|
|
@ -1873,8 +1921,7 @@ mod tests {
|
|||
fn tokenize_pg_regex_match() {
|
||||
let sql = "SELECT col ~ '^a', col ~* '^a', col !~ '^a', col !~* '^a'";
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::make_keyword("SELECT"),
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
|
|
@ -1912,8 +1959,7 @@ mod tests {
|
|||
fn tokenize_quoted_identifier() {
|
||||
let sql = r#" "a "" b" "a """ "c """"" "#;
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, sql);
|
||||
let tokens = tokenizer.tokenize().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap();
|
||||
let expected = vec![
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
Token::make_word(r#"a " b"#, Some('"')),
|
||||
|
|
@ -1926,12 +1972,33 @@ mod tests {
|
|||
compare(expected, tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenize_quoted_identifier_with_no_escape() {
|
||||
let sql = r#" "a "" b" "a """ "c """"" "#;
|
||||
let dialect = GenericDialect {};
|
||||
let tokens = Tokenizer::new(&dialect, sql)
|
||||
.with_unescape(false)
|
||||
.tokenize()
|
||||
.unwrap();
|
||||
let expected = vec![
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
Token::make_word(r#"a "" b"#, Some('"')),
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
Token::make_word(r#"a """#, Some('"')),
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
Token::make_word(r#"c """""#, Some('"')),
|
||||
Token::Whitespace(Whitespace::Space),
|
||||
];
|
||||
compare(expected, tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tokenize_with_location() {
|
||||
let sql = "SELECT a,\n b";
|
||||
let dialect = GenericDialect {};
|
||||
let mut tokenizer = Tokenizer::new(&dialect, sql);
|
||||
let tokens = tokenizer.tokenize_with_location().unwrap();
|
||||
let tokens = Tokenizer::new(&dialect, sql)
|
||||
.tokenize_with_location()
|
||||
.unwrap();
|
||||
let expected = vec![
|
||||
TokenWithLocation::new(Token::make_keyword("SELECT"), 1, 1),
|
||||
TokenWithLocation::new(Token::Whitespace(Whitespace::Space), 1, 7),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue