language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-27 04:19:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

mbe: split Op::Leaf and handle multi-character puncts

Browse source
This commit is contained in:
Ryo Yoshida 2022-12-27 18:18:18 +09:00
parent 47c6c8e2f3
commit ec7148b091
No known key found for this signature in database
GPG key ID: E25698A930586171
6 changed files with 183 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

44
crates/mbe/src/parser.rs
View file

@ -1,7 +1,7 @@
//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
//! trees.
use smallvec::SmallVec;
use smallvec::{smallvec, SmallVec};
use syntax::SmolStr;
use crate::{tt_iter::TtIter, ParseError};
@ -39,7 +39,7 @@ impl MetaTemplate {
let mut src = TtIter::new(tt);
let mut res = Vec::new();
while let Some(first) = src.next() {
while let Some(first) = src.peek_n(0) {
let op = next_op(first, &mut src, mode)?;
res.push(op);
}
@ -54,8 +54,10 @@ pub(crate) enum Op {
Ignore { name: SmolStr, id: tt::TokenId },
Index { depth: u32 },
Repeat { tokens: MetaTemplate, kind: RepeatKind, separator: Option<Separator> },
Leaf(tt::Leaf),
Subtree { tokens: MetaTemplate, delimiter: Option<tt::Delimiter> },
Literal(tt::Literal),
Punct(SmallVec<[tt::Punct; 3]>),
Ident(tt::Ident),
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
@ -124,12 +126,17 @@ enum Mode {
Template,
}
fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Result<Op, ParseError> {
let res = match first {
tt::TokenTree::Leaf(leaf @ tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
fn next_op<'a>(
first_peeked: &tt::TokenTree,
src: &mut TtIter<'a>,
mode: Mode,
) -> Result<Op, ParseError> {
let res = match first_peeked {
tt::TokenTree::Leaf(tt::Leaf::Punct(p @ tt::Punct { char: '$', .. })) => {
src.next().expect("first token already peeked");
// Note that the '$' itself is a valid token inside macro_rules.
let second = match src.next() {
None => return Ok(Op::Leaf(leaf.clone())),
None => return Ok(Op::Punct(smallvec![p.clone()])),
Some(it) => it,
};
match second {
@ -160,7 +167,7 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
tt::TokenTree::Leaf(leaf) => match leaf {
tt::Leaf::Ident(ident) if ident.text == "crate" => {
// We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
Op::Leaf(tt::Leaf::from(tt::Ident { text: "$crate".into(), id: ident.id }))
Op::Ident(tt::Ident { text: "$crate".into(), id: ident.id })
}
tt::Leaf::Ident(ident) => {
let kind = eat_fragment_kind(src, mode)?;
@ -180,7 +187,7 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
"`$$` is not allowed on the pattern side",
))
}
Mode::Template => Op::Leaf(tt::Leaf::Punct(*punct)),
Mode::Template => Op::Punct(smallvec![*punct]),
},
tt::Leaf::Punct(_) | tt::Leaf::Literal(_) => {
return Err(ParseError::expected("expected ident"))
@ -188,8 +195,25 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
},
}
}
tt::TokenTree::Leaf(tt) => Op::Leaf(tt.clone()),
tt::TokenTree::Leaf(tt::Leaf::Literal(it)) => {
src.next().expect("first token already peeked");
Op::Literal(it.clone())
}
tt::TokenTree::Leaf(tt::Leaf::Ident(it)) => {
src.next().expect("first token already peeked");
Op::Ident(it.clone())
}
tt::TokenTree::Leaf(tt::Leaf::Punct(_)) => {
// There's at least one punct so this shouldn't fail.
let puncts = src.expect_glued_punct().unwrap();
Op::Punct(puncts)
}
tt::TokenTree::Subtree(subtree) => {
src.next().expect("first token already peeked");
let tokens = MetaTemplate::parse(subtree, mode)?;
Op::Subtree { tokens, delimiter: subtree.delimiter }
}