mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-08-31 03:07:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

TBD fixup multiline comment tokenization

Browse source
This commit is contained in:
Nickolay Ponomarev 2019-02-11 05:10:53 +03:00
parent 028c613c3f
commit f958e9d3cf
1 changed files with 27 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

38
src/sqltokenizer.rs
View file

@ -482,18 +482,19 @@ impl<'a> Tokenizer<'a> {
// TODO: deal with nested comments
loop {
match chars.next() {
Some(ch) if maybe_closing_comment && ch == '/' => {
break Ok(Some(Token::Whitespace(Whitespace::MultiLineComment(s))));
Some(ch) => {
if maybe_closing_comment {
if ch == '/' {
break Ok(Some(Token::Whitespace(Whitespace::MultiLineComment(s))));
} else {
s.push('*');
}
}
maybe_closing_comment = ch == '*';
if !maybe_closing_comment {
s.push(ch);
}
}
Some(ch) if maybe_closing_comment && ch != '/' => {
maybe_closing_comment = false;
s.push('*');
s.push(ch);
}
Some(ch) if !maybe_closing_comment && ch == '*' => {
maybe_closing_comment = true;
}
Some(ch) => s.push(ch),
None => {
break Err(TokenizerError(
"Unexpected EOF while in a multi-line comment".to_string(),
@ -727,6 +728,21 @@ mod tests {
compare(expected, tokens);
}
#[test]
fn tokenize_multiline_comment_with_even_asterisks() {
let sql = String::from("\n/** Comment **/\n");
let dialect = GenericSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();
let expected = vec![
Token::Whitespace(Whitespace::Newline),
Token::Whitespace(Whitespace::MultiLineComment("* Comment *".to_string())),
Token::Whitespace(Whitespace::Newline),
];
compare(expected, tokens);
}
fn compare(expected: Vec<Token>, actual: Vec<Token>) {
//println!("------------------------------");
//println!("tokens = {:?}", actual);