Add APIs to reuse token buffers in Tokenizer (#1094)

This commit is contained in:
0rphon 2024-01-22 11:46:36 -08:00 committed by GitHub
parent b0b62887a2
commit d72f0a966b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 32 additions and 4 deletions

View file

@ -8711,6 +8711,11 @@ impl<'a> Parser<'a> {
self.expect_token(&Token::RParen)?;
Ok(partitions)
}
/// Consume the parser and return its underlying token buffer
pub fn into_tokens(self) -> Vec<TokenWithLocation> {
self.tokens
}
}
impl Word {