mirror of
https://github.com/apache/datafusion-sqlparser-rs.git
synced 2025-12-23 11:12:51 +00:00
Add lifetime parameter to BorrowedToken for zero-copy tokenization
This change introduces a lifetime parameter 'a to BorrowedToken enum
to prepare for zero-copy tokenization support. This is a foundational
step toward reducing memory allocations during SQL parsing.
Changes:
- Added lifetime parameter to BorrowedToken<'a> enum
- Added _Phantom(Cow<'a, str>) variant to carry the lifetime
- Implemented Visit and VisitMut traits for Cow<'a, str> to support
the visitor pattern with the new lifetime parameter
- Fixed lifetime issues in visitor tests by using tokenized_owned()
instead of tokenize() where owned tokens are required
- Type alias Token = BorrowedToken<'static> maintains backward
compatibility
This commit is contained in:
parent
c6933bb8ec
commit
0f17b327b9
11 changed files with 1724 additions and 1409 deletions
|
|
@ -2045,7 +2045,7 @@ fn parse_pg_returning() {
|
|||
fn test_operator(operator: &str, dialect: &TestedDialects, expected: BinaryOperator) {
|
||||
let operator_tokens =
|
||||
sqlparser::tokenizer::Tokenizer::new(&PostgreSqlDialect {}, &format!("a{operator}b"))
|
||||
.tokenize()
|
||||
.tokenized_owned()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
operator_tokens.len(),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue