language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-11-12 16:58:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Store token trees in contiguous Vec instead of as a tree

Browse source 
I expected this to be faster (due to less allocations and better cache locality), but benchmarked it is not (neither it is slower). Memory usage, however, drops by ~50mb (of `analysis-stats .`). I guess tt construction is just not hot.

This also simplifies using even less memory for token trees by compressing equal span, which I plan to do right after.

Some workflows are more easily expressed with a flat tt, while some are better expressed with a tree. With the right helpers, though (which was mostly a matter of trial and error), even the worst workflows become very easy indeed.
This commit is contained in:
Chayim Refael Friedman 2024-10-18 10:16:08 +03:00
parent 1c6b83852b
commit ceba289f80
50 changed files with 2356 additions and 2286 deletions
Download patch file Download diff file Expand all files Collapse all files

54
crates/syntax-bridge/src/to_parser_input.rs
View file

@ -6,37 +6,34 @@ use std::fmt;
use span::Edition;
use syntax::{SyntaxKind, SyntaxKind::*, T};
use tt::buffer::TokenBuffer;
pub fn to_parser_input<S: Copy + fmt::Debug>(
edition: Edition,
buffer: &TokenBuffer<'_, S>,
buffer: tt::TokenTreesView<'_, S>,
) -> parser::Input {
let mut res = parser::Input::default();
let mut current = buffer.begin();
let mut current = buffer.cursor();
while !current.eof() {
let cursor = current;
let tt = cursor.token_tree();
let tt = current.token_tree();
// Check if it is lifetime
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tt {
if punct.char == '\'' {
let next = cursor.bump();
match next.token_tree() {
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(_ident), _)) => {
current.bump();
match current.token_tree() {
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(_ident))) => {
res.push(LIFETIME_IDENT);
current = next.bump();
current.bump();
continue;
}
_ => panic!("Next token must be ident : {:#?}", next.token_tree()),
_ => panic!("Next token must be ident"),
}
}
}
current = match tt {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
match tt {
Some(tt::TokenTree::Leaf(leaf)) => {
match leaf {
tt::Leaf::Literal(lit) => {
let kind = match lit.kind {
@ -83,9 +80,9 @@ pub fn to_parser_input<S: Copy + fmt::Debug>(
}
}
}
cursor.bump()
current.bump();
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
Some(tt::TokenTree::Subtree(subtree)) => {
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T!['(']),
tt::DelimiterKind::Brace => Some(T!['{']),
@ -94,22 +91,19 @@ pub fn to_parser_input<S: Copy + fmt::Debug>(
} {
res.push(kind);
}
cursor.subtree().unwrap()
current.bump();
}
None => match cursor.end() {
Some(subtree) => {
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.bump()
None => {
let subtree = current.end();
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
None => continue,
},
}
};
}