mirrors/datafusion-sqlparse
1
0
Fork 0
mirror of https://github.com/apache/datafusion-sqlparser-rs.git synced 2025-10-09 21:42:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Start new line if \r in Postgres dialect (#1647)

Browse source
This commit is contained in:
Hans Ott 2025-01-07 18:35:03 +01:00 • committed by GitHub
parent 8cfc46277f
commit 0cd49fb699
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 57 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

63
src/tokenizer.rs
View file

@ -1621,11 +1621,17 @@ impl<'a> Tokenizer<'a> {
// Consume characters until newline // Consume characters until newline
fn tokenize_single_line_comment(&self, chars: &mut State) -> String { fn tokenize_single_line_comment(&self, chars: &mut State) -> String {
let mut comment = peeking_take_while(chars, |ch| ch != '\n'); let mut comment = peeking_take_while(chars, |ch| match ch {
'\n' => false, // Always stop at \n
'\r' if dialect_of!(self is PostgreSqlDialect) => false, // Stop at \r for Postgres
_ => true, // Keep consuming for other characters
});
if let Some(ch) = chars.next() { if let Some(ch) = chars.next() {
assert_eq!(ch, '\n'); assert!(ch == '\n' || ch == '\r');
comment.push(ch); comment.push(ch);
} }
comment comment
} }
@ -2677,17 +2683,62 @@ mod tests {
#[test] #[test]
fn tokenize_comment() { fn tokenize_comment() {
let sql = String::from("0--this is a comment\n1"); let test_cases = vec![
(
String::from("0--this is a comment\n1"),
vec![
Token::Number("0".to_string(), false),
Token::Whitespace(Whitespace::SingleLineComment {
prefix: "--".to_string(),
comment: "this is a comment\n".to_string(),
}),
Token::Number("1".to_string(), false),
],
),
(
String::from("0--this is a comment\r1"),
vec![
Token::Number("0".to_string(), false),
Token::Whitespace(Whitespace::SingleLineComment {
prefix: "--".to_string(),
comment: "this is a comment\r1".to_string(),
}),
],
),
(
String::from("0--this is a comment\r\n1"),
vec![
Token::Number("0".to_string(), false),
Token::Whitespace(Whitespace::SingleLineComment {
prefix: "--".to_string(),
comment: "this is a comment\r\n".to_string(),
}),
Token::Number("1".to_string(), false),
],
),
];
let dialect = GenericDialect {}; let dialect = GenericDialect {};
for (sql, expected) in test_cases {
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
compare(expected, tokens);
}
}
#[test]
fn tokenize_comment_postgres() {
let sql = String::from("1--\r0");
let dialect = PostgreSqlDialect {};
let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap(); let tokens = Tokenizer::new(&dialect, &sql).tokenize().unwrap();
let expected = vec![ let expected = vec![
Token::Number("0".to_string(), false), Token::Number("1".to_string(), false),
Token::Whitespace(Whitespace::SingleLineComment { Token::Whitespace(Whitespace::SingleLineComment {
prefix: "--".to_string(), prefix: "--".to_string(),
comment: "this is a comment\n".to_string(), comment: "\r".to_string(),
}), }),
Token::Number("1".to_string(), false), Token::Number("0".to_string(), false),
]; ];
compare(expected, tokens); compare(expected, tokens);
} }