language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-11-03 21:25:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Store token trees in contiguous Vec instead of as a tree

Browse source 
I expected this to be faster (due to less allocations and better cache locality), but benchmarked it is not (neither it is slower). Memory usage, however, drops by ~50mb (of `analysis-stats .`). I guess tt construction is just not hot.

This also simplifies using even less memory for token trees by compressing equal span, which I plan to do right after.

Some workflows are more easily expressed with a flat tt, while some are better expressed with a tree. With the right helpers, though (which was mostly a matter of trial and error), even the worst workflows become very easy indeed.
This commit is contained in:
Chayim Refael Friedman 2024-10-18 10:16:08 +03:00
parent 1c6b83852b
commit ceba289f80
50 changed files with 2356 additions and 2286 deletions
Download patch file Download diff file Expand all files Collapse all files

21
crates/cfg/src/cfg_expr.rs
View file

@ -45,8 +45,8 @@ impl From<CfgAtom> for CfgExpr {
impl CfgExpr {
#[cfg(feature = "tt")]
pub fn parse<S>(tt: &tt::Subtree<S>) -> CfgExpr {
next_cfg_expr(&mut tt.token_trees.iter()).unwrap_or(CfgExpr::Invalid)
pub fn parse<S: Copy>(tt: &tt::TopSubtree<S>) -> CfgExpr {
next_cfg_expr(&mut tt.iter()).unwrap_or(CfgExpr::Invalid)
}
/// Fold the cfg by querying all basic `Atom` and `KeyValue` predicates.
@ -66,19 +66,19 @@ impl CfgExpr {
}
#[cfg(feature = "tt")]
fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<CfgExpr> {
fn next_cfg_expr<S: Copy>(it: &mut tt::iter::TtIter<'_, S>) -> Option<CfgExpr> {
use intern::sym;
use tt::iter::TtElement;
let name = match it.next() {
None => return None,
Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
Some(TtElement::Leaf(tt::Leaf::Ident(ident))) => ident.sym.clone(),
Some(_) => return Some(CfgExpr::Invalid),
};
// Peek
let ret = match it.as_slice().first() {
Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
match it.as_slice().get(1) {
let ret = match it.peek() {
Some(TtElement::Leaf(tt::Leaf::Punct(punct))) if punct.char == '=' => {
match it.remaining().flat_tokens().get(1) {
Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => {
it.next();
it.next();
@ -87,9 +87,8 @@ fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<C
_ => return Some(CfgExpr::Invalid),
}
}
Some(tt::TokenTree::Subtree(subtree)) => {
Some(TtElement::Subtree(_, mut sub_it)) => {
it.next();
let mut sub_it = subtree.token_trees.iter();
let mut subs = std::iter::from_fn(|| next_cfg_expr(&mut sub_it));
match name {
s if s == sym::all => CfgExpr::All(subs.collect()),
@ -104,7 +103,7 @@ fn next_cfg_expr<S>(it: &mut std::slice::Iter<'_, tt::TokenTree<S>>) -> Option<C
};
// Eat comma separator
if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = it.as_slice().first() {
if let Some(TtElement::Leaf(tt::Leaf::Punct(punct))) = it.peek() {
if punct.char == ',' {
it.next();
}