language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-27 04:19:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Remove Delimiter::DUMMY_INVISIBLE

Browse source
This commit is contained in:
Lukas Wirth 2023-12-20 14:00:14 +01:00
parent 2c6ce480e3
commit 7b804552a5
20 changed files with 170 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

7
crates/mbe/src/benchmark.rs
View file

@ -67,8 +67,11 @@ fn macro_rules_fixtures_tt() -> FxHashMap<String, tt::Subtree<DummyTestSpanData>
.filter_map(ast::MacroRules::cast)
.map(|rule| {
let id = rule.name().unwrap().to_string();
let def_tt =
syntax_node_to_token_tree(rule.token_tree().unwrap().syntax(), DummyTestSpanMap);
let def_tt = syntax_node_to_token_tree(
rule.token_tree().unwrap().syntax(),
DummyTestSpanMap,
DUMMY,
);
(id, def_tt)
})
.collect()

6
crates/mbe/src/expander.rs
View file

@ -56,7 +56,10 @@ pub(crate) fn expand_rules<S: Span>(
ExpandResult { value, err: match_.err.or(transcribe_err) }
} else {
ExpandResult::new(
tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![] },
tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(call_site),
token_trees: vec![],
},
ExpandError::NoMatchingRule,
)
}
@ -128,6 +131,7 @@ enum Binding<S> {
#[derive(Debug, Clone, PartialEq, Eq)]
enum Fragment<S> {
Empty,
/// token fragments are just copy-pasted into the output
Tokens(tt::TokenTree<S>),
/// Expr ast fragments are surrounded with `()` on insertion to preserve

33
crates/mbe/src/expander/matcher.rs
View file

@ -63,7 +63,7 @@ use std::rc::Rc;
use smallvec::{smallvec, SmallVec};
use syntax::SmolStr;
use tt::Span;
use tt::{DelimSpan, Span};
use crate::{
expander::{Binding, Bindings, ExpandResult, Fragment},
@ -74,11 +74,7 @@ use crate::{
impl<S: Span> Bindings<S> {
fn push_optional(&mut self, name: &SmolStr) {
// FIXME: Do we have a better way to represent an empty token ?
// Insert an empty subtree for empty token
let tt =
tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![] }.into();
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
self.inner.insert(name.clone(), Binding::Fragment(Fragment::Empty));
}
fn push_empty(&mut self, name: &SmolStr) {
@ -387,6 +383,7 @@ fn match_loop_inner<'t, S: Span>(
eof_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
error_items: &mut SmallVec<[MatchState<'t, S>; 1]>,
is_2021: bool,
delim_span: tt::DelimSpan<S>,
) {
macro_rules! try_push {
($items: expr, $it:expr) => {
@ -474,7 +471,7 @@ fn match_loop_inner<'t, S: Span>(
cur_items.push(new_item);
}
cur_items.push(MatchState {
dot: tokens.iter_delimited(None),
dot: tokens.iter_delimited(delim_span),
stack: Default::default(),
up: Some(Box::new(item)),
sep: separator.clone(),
@ -489,7 +486,7 @@ fn match_loop_inner<'t, S: Span>(
if let Ok(subtree) = src.clone().expect_subtree() {
if subtree.delimiter.kind == delimiter.kind {
item.stack.push(item.dot);
item.dot = tokens.iter_delimited(Some(*delimiter));
item.dot = tokens.iter_delimited_with(*delimiter);
cur_items.push(item);
}
}
@ -497,7 +494,7 @@ fn match_loop_inner<'t, S: Span>(
OpDelimited::Op(Op::Var { kind, name, .. }) => {
if let &Some(kind) = kind {
let mut fork = src.clone();
let match_res = match_meta_var(kind, &mut fork, is_2021);
let match_res = match_meta_var(kind, &mut fork, is_2021, delim_span);
match match_res.err {
None => {
// Some meta variables are optional (e.g. vis)
@ -611,6 +608,7 @@ fn match_loop_inner<'t, S: Span>(
}
fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021: bool) -> Match<S> {
let span = src.delimiter.delim_span();
let mut src = TtIter::new(src);
let mut stack: SmallVec<[TtIter<'_, S>; 1]> = SmallVec::new();
let mut res = Match::default();
@ -619,7 +617,7 @@ fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021:
let mut bindings_builder = BindingsBuilder::default();
let mut cur_items = smallvec![MatchState {
dot: pattern.iter_delimited(None),
dot: pattern.iter_delimited(span),
stack: Default::default(),
up: None,
sep: None,
@ -650,6 +648,7 @@ fn match_loop<S: Span>(pattern: &MetaTemplate<S>, src: &tt::Subtree<S>, is_2021:
&mut eof_items,
&mut error_items,
is_2021,
span,
);
stdx::always!(cur_items.is_empty());
@ -763,12 +762,13 @@ fn match_meta_var<S: Span>(
kind: MetaVarKind,
input: &mut TtIter<'_, S>,
is_2021: bool,
delim_span: DelimSpan<S>,
) -> ExpandResult<Option<Fragment<S>>> {
let fragment = match kind {
MetaVarKind::Path => {
return input
.expect_fragment(parser::PrefixEntryPoint::Path)
.map(|it| it.map(tt::TokenTree::subtree_or_wrap).map(Fragment::Path));
return input.expect_fragment(parser::PrefixEntryPoint::Path).map(|it| {
it.map(|it| tt::TokenTree::subtree_or_wrap(it, delim_span)).map(Fragment::Path)
});
}
MetaVarKind::Ty => parser::PrefixEntryPoint::Ty,
MetaVarKind::Pat if is_2021 => parser::PrefixEntryPoint::PatTop,
@ -860,11 +860,14 @@ fn collect_vars<S: Span>(collector_fun: &mut impl FnMut(SmolStr), pattern: &Meta
}
}
impl<S: Span> MetaTemplate<S> {
fn iter_delimited(&self, delimited: Option<tt::Delimiter<S>>) -> OpDelimitedIter<'_, S> {
fn iter_delimited_with(&self, delimiter: tt::Delimiter<S>) -> OpDelimitedIter<'_, S> {
OpDelimitedIter { inner: &self.0, idx: 0, delimited: delimiter }
}
fn iter_delimited(&self, span: tt::DelimSpan<S>) -> OpDelimitedIter<'_, S> {
OpDelimitedIter {
inner: &self.0,
idx: 0,
delimited: delimited.unwrap_or(tt::Delimiter::DUMMY_INVISIBLE),
delimited: tt::Delimiter::invisible_delim_spanned(span),
}
}
}

20
crates/mbe/src/expander/transcriber.rs
View file

@ -59,12 +59,12 @@ impl<S: Span> Bindings<S> {
token_trees: token_trees.clone(),
};
Ok(match f {
Fragment::Tokens(_) => unreachable!(),
Fragment::Tokens(_) | Fragment::Empty => unreachable!(),
Fragment::Expr(_) => Fragment::Expr,
Fragment::Path(_) => Fragment::Path,
}(subtree))
}
Binding::Fragment(it @ Fragment::Tokens(_)) => Ok(it.clone()),
Binding::Fragment(it @ (Fragment::Tokens(_) | Fragment::Empty)) => Ok(it.clone()),
// emit some reasonable default expansion for missing bindings,
// this gives better recovery than emitting the `$fragment-name` verbatim
Binding::Missing(it) => Ok({
@ -87,10 +87,7 @@ impl<S: Span> Bindings<S> {
})),
// FIXME: Meta and Item should get proper defaults
MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {
Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::DUMMY_INVISIBLE,
token_trees: vec![],
}))
Fragment::Empty
}
MetaVarKind::Path
| MetaVarKind::Ty
@ -351,7 +348,7 @@ fn expand_var<S: Span>(
// ```
// We just treat it a normal tokens
let tt = tt::Subtree {
delimiter: tt::Delimiter::DUMMY_INVISIBLE,
delimiter: tt::Delimiter::invisible_spanned(id),
token_trees: vec![
tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id })
.into(),
@ -422,7 +419,7 @@ fn expand_repeat<S: Span>(
continue;
}
t.delimiter = tt::Delimiter::DUMMY_INVISIBLE;
t.delimiter.kind = tt::DelimiterKind::Invisible;
push_subtree(&mut buf, t);
if let Some(sep) = separator {
@ -456,7 +453,11 @@ fn expand_repeat<S: Span>(
// Check if it is a single token subtree without any delimiter
// e.g {Delimiter:None> ['>'] /Delimiter:None>}
let tt = tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: buf }.into();
let tt = tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(ctx.call_site),
token_trees: buf,
}
.into();
if RepeatKind::OneOrMore == kind && counter == 0 {
return ExpandResult {
@ -479,6 +480,7 @@ fn push_fragment<S: Span>(
}
Fragment::Path(tt) => fix_up_and_push_path_tt(ctx, buf, tt),
Fragment::Tokens(tt) => buf.push(tt),
Fragment::Empty => (),
}
}

37
crates/mbe/src/syntax_bridge.rs
View file

@ -78,13 +78,14 @@ pub(crate) mod dummy_test_span_utils {
pub fn syntax_node_to_token_tree<Ctx, SpanMap>(
node: &SyntaxNode,
map: SpanMap,
span: SpanData<Ctx>,
) -> tt::Subtree<SpanData<Ctx>>
where
SpanData<Ctx>: Span,
Ctx: SyntaxContext,
SpanMap: SpanMapper<SpanData<Ctx>>,
{
let mut c = Converter::new(node, map, Default::default(), Default::default());
let mut c = Converter::new(node, map, Default::default(), Default::default(), span);
convert_tokens(&mut c)
}
@ -96,13 +97,14 @@ pub fn syntax_node_to_token_tree_modified<Ctx, SpanMap>(
map: SpanMap,
append: FxHashMap<SyntaxElement, Vec<tt::Leaf<SpanData<Ctx>>>>,
remove: FxHashSet<SyntaxNode>,
call_site: SpanData<Ctx>,
) -> tt::Subtree<SpanData<Ctx>>
where
SpanMap: SpanMapper<SpanData<Ctx>>,
SpanData<Ctx>: Span,
Ctx: SyntaxContext,
{
let mut c = Converter::new(node, map, append, remove);
let mut c = Converter::new(node, map, append, remove, call_site);
convert_tokens(&mut c)
}
@ -187,7 +189,11 @@ where
}
/// Split token tree with separate expr: $($e:expr)SEP*
pub fn parse_exprs_with_sep<S: Span>(tt: &tt::Subtree<S>, sep: char) -> Vec<tt::Subtree<S>> {
pub fn parse_exprs_with_sep<S: Span>(
tt: &tt::Subtree<S>,
sep: char,
span: S,
) -> Vec<tt::Subtree<S>> {
if tt.token_trees.is_empty() {
return Vec::new();
}
@ -200,7 +206,7 @@ pub fn parse_exprs_with_sep<S: Span>(tt: &tt::Subtree<S>, sep: char) -> Vec<tt::
res.push(match expanded.value {
None => break,
Some(tt) => tt.subtree_or_wrap(),
Some(tt) => tt.subtree_or_wrap(tt::DelimSpan { open: span, close: span }),
});
let mut fork = iter.clone();
@ -212,7 +218,7 @@ pub fn parse_exprs_with_sep<S: Span>(tt: &tt::Subtree<S>, sep: char) -> Vec<tt::
if iter.peek_n(0).is_some() {
res.push(tt::Subtree {
delimiter: tt::Delimiter::DUMMY_INVISIBLE,
delimiter: tt::Delimiter::invisible_spanned(span),
token_trees: iter.cloned().collect(),
});
}
@ -225,7 +231,10 @@ where
C: TokenConverter<S>,
S: Span,
{
let entry = tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: vec![] };
let entry = tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(conv.call_site()),
token_trees: vec![],
};
let mut stack = NonEmptyVec::new(entry);
while let Some((token, abs_range)) = conv.bump() {
@ -490,6 +499,8 @@ trait TokenConverter<S>: Sized {
fn peek(&self) -> Option<Self::Token>;
fn span_for(&self, range: TextRange) -> S;
fn call_site(&self) -> S;
}
impl<S, Ctx> SrcToken<RawConverter<'_, Ctx>, S> for usize {
@ -557,6 +568,10 @@ where
fn span_for(&self, range: TextRange) -> SpanData<Ctx> {
SpanData { range, anchor: self.anchor, ctx: self.ctx }
}
fn call_site(&self) -> SpanData<Ctx> {
SpanData { range: TextRange::empty(0.into()), anchor: self.anchor, ctx: self.ctx }
}
}
impl<S> TokenConverter<S> for StaticRawConverter<'_, S>
@ -592,6 +607,10 @@ where
fn span_for(&self, _: TextRange) -> S {
self.span
}
fn call_site(&self) -> S {
self.span
}
}
struct Converter<SpanMap, S> {
@ -604,6 +623,7 @@ struct Converter<SpanMap, S> {
map: SpanMap,
append: FxHashMap<SyntaxElement, Vec<tt::Leaf<S>>>,
remove: FxHashSet<SyntaxNode>,
call_site: S,
}
impl<SpanMap, S> Converter<SpanMap, S> {
@ -612,6 +632,7 @@ impl<SpanMap, S> Converter<SpanMap, S> {
map: SpanMap,
append: FxHashMap<SyntaxElement, Vec<tt::Leaf<S>>>,
remove: FxHashSet<SyntaxNode>,
call_site: S,
) -> Self {
let mut this = Converter {
current: None,
@ -621,6 +642,7 @@ impl<SpanMap, S> Converter<SpanMap, S> {
map,
append,
remove,
call_site,
current_leafs: vec![],
};
let first = this.next_token();
@ -780,6 +802,9 @@ where
fn span_for(&self, range: TextRange) -> S {
self.map.span_for(range)
}
fn call_site(&self) -> S {
self.call_site
}
}
struct TtTreeSink<'a, Ctx>

4
crates/mbe/src/syntax_bridge/tests.rs
View file

@ -7,11 +7,11 @@ use tt::{
Leaf, Punct, Spacing,
};
use crate::{syntax_node_to_token_tree, DummyTestSpanData, DummyTestSpanMap};
use crate::{syntax_node_to_token_tree, DummyTestSpanData, DummyTestSpanMap, DUMMY};
fn check_punct_spacing(fixture: &str) {
let source_file = ast::SourceFile::parse(fixture).ok().unwrap();
let subtree = syntax_node_to_token_tree(source_file.syntax(), DummyTestSpanMap);
let subtree = syntax_node_to_token_tree(source_file.syntax(), DummyTestSpanMap, DUMMY);
let mut annotations: HashMap<_, _> = extract_annotations(fixture)
.into_iter()
.map(|(range, annotation)| {

8
crates/mbe/src/tt_iter.rs
View file

@ -176,10 +176,10 @@ impl<'a, S: Span> TtIter<'a, S> {
}
self.inner = self.inner.as_slice()[res.len()..].iter();
let res = match res.len() {
0 | 1 => res.pop(),
_ => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::DUMMY_INVISIBLE,
let res = match &*res {
[] | [_] => res.pop(),
[first, ..] => Some(tt::TokenTree::Subtree(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(first.first_span()),
token_trees: res,
})),
};