Rename ra_mbe -> mbe

crates/mbe/src/lib.rs

@@ -0,0 +1,278 @@
+//! `mbe` (short for Macro By Example) crate contains code for handling
+//! `macro_rules` macros. It uses `TokenTree` (from `tt` package) as the
+//! interface, although it contains some code to bridge `SyntaxNode`s and
+//! `TokenTree`s as well!
+
+mod parser;
+mod mbe_expander;
+mod syntax_bridge;
+mod tt_iter;
+mod subtree_source;
+
+#[cfg(test)]
+mod tests;
+
+pub use tt::{Delimiter, Punct};
+
+use crate::{
+    parser::{parse_pattern, Op},
+    tt_iter::TtIter,
+};
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum ParseError {
+    Expected(String),
+    RepetitionEmtpyTokenTree,
+}
+
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub enum ExpandError {
+    NoMatchingRule,
+    UnexpectedToken,
+    BindingError(String),
+    ConversionError,
+    InvalidRepeat,
+    ProcMacroError(tt::ExpansionError),
+}
+
+impl From<tt::ExpansionError> for ExpandError {
+    fn from(it: tt::ExpansionError) -> Self {
+        ExpandError::ProcMacroError(it)
+    }
+}
+
+pub use crate::syntax_bridge::{
+    ast_to_token_tree, parse_to_token_tree, syntax_node_to_token_tree, token_tree_to_syntax_node,
+    TokenMap,
+};
+
+/// This struct contains AST for a single `macro_rules` definition. What might
+/// be very confusing is that AST has almost exactly the same shape as
+/// `tt::TokenTree`, but there's a crucial difference: in macro rules, `$ident`
+/// and `$()*` have special meaning (see `Var` and `Repeat` data structures)
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct MacroRules {
+    rules: Vec<Rule>,
+    /// Highest id of the token we have in TokenMap
+    shift: Shift,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct Rule {
+    lhs: tt::Subtree,
+    rhs: tt::Subtree,
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct Shift(u32);
+
+impl Shift {
+    fn new(tt: &tt::Subtree) -> Shift {
+        // Note that TokenId is started from zero,
+        // We have to add 1 to prevent duplication.
+        let value = max_id(tt).map_or(0, |it| it + 1);
+        return Shift(value);
+
+        // Find the max token id inside a subtree
+        fn max_id(subtree: &tt::Subtree) -> Option<u32> {
+            subtree
+                .token_trees
+                .iter()
+                .filter_map(|tt| match tt {
+                    tt::TokenTree::Subtree(subtree) => {
+                        let tree_id = max_id(subtree);
+                        match subtree.delimiter {
+                            Some(it) if it.id != tt::TokenId::unspecified() => {
+                                Some(tree_id.map_or(it.id.0, |t| t.max(it.id.0)))
+                            }
+                            _ => tree_id,
+                        }
+                    }
+                    tt::TokenTree::Leaf(tt::Leaf::Ident(ident))
+                        if ident.id != tt::TokenId::unspecified() =>
+                    {
+                        Some(ident.id.0)
+                    }
+                    _ => None,
+                })
+                .max()
+        }
+    }
+
+    /// Shift given TokenTree token id
+    fn shift_all(self, tt: &mut tt::Subtree) {
+        for t in tt.token_trees.iter_mut() {
+            match t {
+                tt::TokenTree::Leaf(leaf) => match leaf {
+                    tt::Leaf::Ident(ident) => ident.id = self.shift(ident.id),
+                    tt::Leaf::Punct(punct) => punct.id = self.shift(punct.id),
+                    tt::Leaf::Literal(lit) => lit.id = self.shift(lit.id),
+                },
+                tt::TokenTree::Subtree(tt) => {
+                    if let Some(it) = tt.delimiter.as_mut() {
+                        it.id = self.shift(it.id);
+                    };
+                    self.shift_all(tt)
+                }
+            }
+        }
+    }
+
+    fn shift(self, id: tt::TokenId) -> tt::TokenId {
+        if id == tt::TokenId::unspecified() {
+            return id;
+        }
+        tt::TokenId(id.0 + self.0)
+    }
+
+    fn unshift(self, id: tt::TokenId) -> Option<tt::TokenId> {
+        id.0.checked_sub(self.0).map(tt::TokenId)
+    }
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum Origin {
+    Def,
+    Call,
+}
+
+impl MacroRules {
+    pub fn parse(tt: &tt::Subtree) -> Result<MacroRules, ParseError> {
+        // Note: this parsing can be implemented using mbe machinery itself, by
+        // matching against `$($lhs:tt => $rhs:tt);*` pattern, but implementing
+        // manually seems easier.
+        let mut src = TtIter::new(tt);
+        let mut rules = Vec::new();
+        while src.len() > 0 {
+            let rule = Rule::parse(&mut src)?;
+            rules.push(rule);
+            if let Err(()) = src.expect_char(';') {
+                if src.len() > 0 {
+                    return Err(ParseError::Expected("expected `:`".to_string()));
+                }
+                break;
+            }
+        }
+
+        for rule in rules.iter() {
+            validate(&rule.lhs)?;
+        }
+
+        Ok(MacroRules { rules, shift: Shift::new(tt) })
+    }
+
+    pub fn expand(&self, tt: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+        // apply shift
+        let mut tt = tt.clone();
+        self.shift.shift_all(&mut tt);
+        mbe_expander::expand(self, &tt)
+    }
+
+    pub fn map_id_down(&self, id: tt::TokenId) -> tt::TokenId {
+        self.shift.shift(id)
+    }
+
+    pub fn map_id_up(&self, id: tt::TokenId) -> (tt::TokenId, Origin) {
+        match self.shift.unshift(id) {
+            Some(id) => (id, Origin::Call),
+            None => (id, Origin::Def),
+        }
+    }
+}
+
+impl Rule {
+    fn parse(src: &mut TtIter) -> Result<Rule, ParseError> {
+        let mut lhs = src
+            .expect_subtree()
+            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
+            .clone();
+        lhs.delimiter = None;
+        src.expect_char('=').map_err(|()| ParseError::Expected("expected `=`".to_string()))?;
+        src.expect_char('>').map_err(|()| ParseError::Expected("expected `>`".to_string()))?;
+        let mut rhs = src
+            .expect_subtree()
+            .map_err(|()| ParseError::Expected("expected subtree".to_string()))?
+            .clone();
+        rhs.delimiter = None;
+        Ok(crate::Rule { lhs, rhs })
+    }
+}
+
+fn to_parse_error(e: ExpandError) -> ParseError {
+    let msg = match e {
+        ExpandError::InvalidRepeat => "invalid repeat".to_string(),
+        _ => "invalid macro definition".to_string(),
+    };
+    ParseError::Expected(msg)
+}
+
+fn validate(pattern: &tt::Subtree) -> Result<(), ParseError> {
+    for op in parse_pattern(pattern) {
+        let op = op.map_err(to_parse_error)?;
+
+        match op {
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) => validate(subtree)?,
+            Op::Repeat { subtree, separator, .. } => {
+                // Checks that no repetition which could match an empty token
+                // https://github.com/rust-lang/rust/blob/a58b1ed44f5e06976de2bdc4d7dc81c36a96934f/src/librustc_expand/mbe/macro_rules.rs#L558
+
+                if separator.is_none() {
+                    if parse_pattern(subtree).all(|child_op| {
+                        match child_op.map_err(to_parse_error) {
+                            Ok(Op::Var { kind, .. }) => {
+                                // vis is optional
+                                if kind.map_or(false, |it| it == "vis") {
+                                    return true;
+                                }
+                            }
+                            Ok(Op::Repeat { kind, .. }) => {
+                                return matches!(
+                                    kind,
+                                    parser::RepeatKind::ZeroOrMore | parser::RepeatKind::ZeroOrOne
+                                )
+                            }
+                            _ => {}
+                        }
+                        false
+                    }) {
+                        return Err(ParseError::RepetitionEmtpyTokenTree);
+                    }
+                }
+                validate(subtree)?
+            }
+            _ => (),
+        }
+    }
+    Ok(())
+}
+
+#[derive(Debug)]
+pub struct ExpandResult<T>(pub T, pub Option<ExpandError>);
+
+impl<T> ExpandResult<T> {
+    pub fn ok(t: T) -> ExpandResult<T> {
+        ExpandResult(t, None)
+    }
+
+    pub fn only_err(err: ExpandError) -> ExpandResult<T>
+    where
+        T: Default,
+    {
+        ExpandResult(Default::default(), Some(err))
+    }
+
+    pub fn map<U>(self, f: impl FnOnce(T) -> U) -> ExpandResult<U> {
+        ExpandResult(f(self.0), self.1)
+    }
+
+    pub fn result(self) -> Result<T, ExpandError> {
+        self.1.map(Err).unwrap_or(Ok(self.0))
+    }
+}
+
+impl<T: Default> From<Result<T, ExpandError>> for ExpandResult<T> {
+    fn from(result: Result<T, ExpandError>) -> ExpandResult<T> {
+        result
+            .map_or_else(|e| ExpandResult(Default::default(), Some(e)), |it| ExpandResult(it, None))
+    }
+}

crates/mbe/src/mbe_expander.rs

@@ -0,0 +1,180 @@
+//! This module takes a (parsed) definition of `macro_rules` invocation, a
+//! `tt::TokenTree` representing an argument of macro invocation, and produces a
+//! `tt::TokenTree` for the result of the expansion.
+
+mod matcher;
+mod transcriber;
+
+use rustc_hash::FxHashMap;
+use syntax::SmolStr;
+
+use crate::{ExpandError, ExpandResult};
+
+pub(crate) fn expand(rules: &crate::MacroRules, input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+    expand_rules(&rules.rules, input)
+}
+
+fn expand_rules(rules: &[crate::Rule], input: &tt::Subtree) -> ExpandResult<tt::Subtree> {
+    let mut match_: Option<(matcher::Match, &crate::Rule)> = None;
+    for rule in rules {
+        let new_match = match matcher::match_(&rule.lhs, input) {
+            Ok(m) => m,
+            Err(_e) => {
+                // error in pattern parsing
+                continue;
+            }
+        };
+        if new_match.err.is_none() {
+            // If we find a rule that applies without errors, we're done.
+            // Unconditionally returning the transcription here makes the
+            // `test_repeat_bad_var` test fail.
+            let ExpandResult(res, transcribe_err) =
+                transcriber::transcribe(&rule.rhs, &new_match.bindings);
+            if transcribe_err.is_none() {
+                return ExpandResult::ok(res);
+            }
+        }
+        // Use the rule if we matched more tokens, or had fewer errors
+        if let Some((prev_match, _)) = &match_ {
+            if (new_match.unmatched_tts, new_match.err_count)
+                < (prev_match.unmatched_tts, prev_match.err_count)
+            {
+                match_ = Some((new_match, rule));
+            }
+        } else {
+            match_ = Some((new_match, rule));
+        }
+    }
+    if let Some((match_, rule)) = match_ {
+        // if we got here, there was no match without errors
+        let ExpandResult(result, transcribe_err) =
+            transcriber::transcribe(&rule.rhs, &match_.bindings);
+        ExpandResult(result, match_.err.or(transcribe_err))
+    } else {
+        ExpandResult(tt::Subtree::default(), Some(ExpandError::NoMatchingRule))
+    }
+}
+
+/// The actual algorithm for expansion is not too hard, but is pretty tricky.
+/// `Bindings` structure is the key to understanding what we are doing here.
+///
+/// On the high level, it stores mapping from meta variables to the bits of
+/// syntax it should be substituted with. For example, if `$e:expr` is matched
+/// with `1 + 1` by macro_rules, the `Binding` will store `$e -> 1 + 1`.
+///
+/// The tricky bit is dealing with repetitions (`$()*`). Consider this example:
+///
+/// ```not_rust
+/// macro_rules! foo {
+///     ($($ i:ident $($ e:expr),*);*) => {
+///         $(fn $ i() { $($ e);*; })*
+///     }
+/// }
+/// foo! { foo 1,2,3; bar 4,5,6 }
+/// ```
+///
+/// Here, the `$i` meta variable is matched first with `foo` and then with
+/// `bar`, and `$e` is matched in turn with `1`, `2`, `3`, `4`, `5`, `6`.
+///
+/// To represent such "multi-mappings", we use a recursive structures: we map
+/// variables not to values, but to *lists* of values or other lists (that is,
+/// to the trees).
+///
+/// For the above example, the bindings would store
+///
+/// ```not_rust
+/// i -> [foo, bar]
+/// e -> [[1, 2, 3], [4, 5, 6]]
+/// ```
+///
+/// We construct `Bindings` in the `match_lhs`. The interesting case is
+/// `TokenTree::Repeat`, where we use `push_nested` to create the desired
+/// nesting structure.
+///
+/// The other side of the puzzle is `expand_subtree`, where we use the bindings
+/// to substitute meta variables in the output template. When expanding, we
+/// maintain a `nesting` stack of indices which tells us which occurrence from
+/// the `Bindings` we should take. We push to the stack when we enter a
+/// repetition.
+///
+/// In other words, `Bindings` is a *multi* mapping from `SmolStr` to
+/// `tt::TokenTree`, where the index to select a particular `TokenTree` among
+/// many is not a plain `usize`, but an `&[usize]`.
+#[derive(Debug, Default)]
+struct Bindings {
+    inner: FxHashMap<SmolStr, Binding>,
+}
+
+#[derive(Debug)]
+enum Binding {
+    Fragment(Fragment),
+    Nested(Vec<Binding>),
+    Empty,
+}
+
+#[derive(Debug, Clone)]
+enum Fragment {
+    /// token fragments are just copy-pasted into the output
+    Tokens(tt::TokenTree),
+    /// Ast fragments are inserted with fake delimiters, so as to make things
+    /// like `$i * 2` where `$i = 1 + 1` work as expectd.
+    Ast(tt::TokenTree),
+}
+
+#[cfg(test)]
+mod tests {
+    use syntax::{ast, AstNode};
+
+    use super::*;
+    use crate::ast_to_token_tree;
+
+    #[test]
+    fn test_expand_rule() {
+        assert_err(
+            "($($i:ident);*) => ($i)",
+            "foo!{a}",
+            ExpandError::BindingError(String::from(
+                "expected simple binding, found nested binding `i`",
+            )),
+        );
+
+        // FIXME:
+        // Add an err test case for ($($i:ident)) => ($())
+    }
+
+    fn assert_err(macro_body: &str, invocation: &str, err: ExpandError) {
+        assert_eq!(expand_first(&create_rules(&format_macro(macro_body)), invocation).1, Some(err));
+    }
+
+    fn format_macro(macro_body: &str) -> String {
+        format!(
+            "
+        macro_rules! foo {{
+            {}
+        }}
+",
+            macro_body
+        )
+    }
+
+    fn create_rules(macro_definition: &str) -> crate::MacroRules {
+        let source_file = ast::SourceFile::parse(macro_definition).ok().unwrap();
+        let macro_definition =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (definition_tt, _) =
+            ast_to_token_tree(&macro_definition.token_tree().unwrap()).unwrap();
+        crate::MacroRules::parse(&definition_tt).unwrap()
+    }
+
+    fn expand_first(rules: &crate::MacroRules, invocation: &str) -> ExpandResult<tt::Subtree> {
+        let source_file = ast::SourceFile::parse(invocation).ok().unwrap();
+        let macro_invocation =
+            source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+
+        let (invocation_tt, _) =
+            ast_to_token_tree(&macro_invocation.token_tree().unwrap()).unwrap();
+
+        expand_rules(&rules.rules, &invocation_tt)
+    }
+}

crates/mbe/src/mbe_expander/matcher.rs

@@ -0,0 +1,477 @@
+//! FIXME: write short doc here
+
+use crate::{
+    mbe_expander::{Binding, Bindings, Fragment},
+    parser::{parse_pattern, Op, RepeatKind, Separator},
+    subtree_source::SubtreeTokenSource,
+    tt_iter::TtIter,
+    ExpandError,
+};
+
+use super::ExpandResult;
+use parser::{FragmentKind::*, TreeSink};
+use syntax::{SmolStr, SyntaxKind};
+use tt::buffer::{Cursor, TokenBuffer};
+
+impl Bindings {
+    fn push_optional(&mut self, name: &SmolStr) {
+        // FIXME: Do we have a better way to represent an empty token ?
+        // Insert an empty subtree for empty token
+        let tt = tt::Subtree::default().into();
+        self.inner.insert(name.clone(), Binding::Fragment(Fragment::Tokens(tt)));
+    }
+
+    fn push_empty(&mut self, name: &SmolStr) {
+        self.inner.insert(name.clone(), Binding::Empty);
+    }
+
+    fn push_nested(&mut self, idx: usize, nested: Bindings) -> Result<(), ExpandError> {
+        for (key, value) in nested.inner {
+            if !self.inner.contains_key(&key) {
+                self.inner.insert(key.clone(), Binding::Nested(Vec::new()));
+            }
+            match self.inner.get_mut(&key) {
+                Some(Binding::Nested(it)) => {
+                    // insert empty nested bindings before this one
+                    while it.len() < idx {
+                        it.push(Binding::Nested(vec![]));
+                    }
+                    it.push(value);
+                }
+                _ => {
+                    return Err(ExpandError::BindingError(format!(
+                        "could not find binding `{}`",
+                        key
+                    )));
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+macro_rules! err {
+    () => {
+        ExpandError::BindingError(format!(""))
+    };
+    ($($tt:tt)*) => {
+        ExpandError::BindingError(format!($($tt)*))
+    };
+}
+
+#[derive(Debug, Default)]
+pub(super) struct Match {
+    pub bindings: Bindings,
+    /// We currently just keep the first error and count the rest to compare matches.
+    pub err: Option<ExpandError>,
+    pub err_count: usize,
+    /// How many top-level token trees were left to match.
+    pub unmatched_tts: usize,
+}
+
+impl Match {
+    pub fn add_err(&mut self, err: ExpandError) {
+        let prev_err = self.err.take();
+        self.err = prev_err.or(Some(err));
+        self.err_count += 1;
+    }
+}
+
+// General note: These functions have two channels to return errors, a `Result`
+// return value and the `&mut Match`. The returned Result is for pattern parsing
+// errors; if a branch of the macro definition doesn't parse, it doesn't make
+// sense to try using it. Matching errors are added to the `Match`. It might
+// make sense to make pattern parsing a separate step?
+
+pub(super) fn match_(pattern: &tt::Subtree, src: &tt::Subtree) -> Result<Match, ExpandError> {
+    assert!(pattern.delimiter == None);
+
+    let mut res = Match::default();
+    let mut src = TtIter::new(src);
+
+    match_subtree(&mut res, pattern, &mut src)?;
+
+    if src.len() > 0 {
+        res.unmatched_tts += src.len();
+        res.add_err(err!("leftover tokens"));
+    }
+
+    Ok(res)
+}
+
+fn match_subtree(
+    res: &mut Match,
+    pattern: &tt::Subtree,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::TokenTree(tt::TokenTree::Leaf(lhs)) => {
+                let rhs = match src.expect_leaf() {
+                    Ok(l) => l,
+                    Err(()) => {
+                        res.add_err(err!("expected leaf: `{}`", lhs));
+                        continue;
+                    }
+                };
+                match (lhs, rhs) {
+                    (
+                        tt::Leaf::Punct(tt::Punct { char: lhs, .. }),
+                        tt::Leaf::Punct(tt::Punct { char: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Ident(tt::Ident { text: lhs, .. }),
+                        tt::Leaf::Ident(tt::Ident { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    (
+                        tt::Leaf::Literal(tt::Literal { text: lhs, .. }),
+                        tt::Leaf::Literal(tt::Literal { text: rhs, .. }),
+                    ) if lhs == rhs => (),
+                    _ => {
+                        res.add_err(ExpandError::UnexpectedToken);
+                    }
+                }
+            }
+            Op::TokenTree(tt::TokenTree::Subtree(lhs)) => {
+                let rhs = match src.expect_subtree() {
+                    Ok(s) => s,
+                    Err(()) => {
+                        res.add_err(err!("expected subtree"));
+                        continue;
+                    }
+                };
+                if lhs.delimiter_kind() != rhs.delimiter_kind() {
+                    res.add_err(err!("mismatched delimiter"));
+                    continue;
+                }
+                let mut src = TtIter::new(rhs);
+                match_subtree(res, lhs, &mut src)?;
+                if src.len() > 0 {
+                    res.add_err(err!("leftover tokens"));
+                }
+            }
+            Op::Var { name, kind } => {
+                let kind = match kind {
+                    Some(k) => k,
+                    None => {
+                        res.add_err(ExpandError::UnexpectedToken);
+                        continue;
+                    }
+                };
+                let ExpandResult(matched, match_err) = match_meta_var(kind.as_str(), src);
+                match matched {
+                    Some(fragment) => {
+                        res.bindings.inner.insert(name.clone(), Binding::Fragment(fragment));
+                    }
+                    None if match_err.is_none() => res.bindings.push_optional(name),
+                    _ => {}
+                }
+                if let Some(err) = match_err {
+                    res.add_err(err);
+                }
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                match_repeat(res, subtree, kind, separator, src)?;
+            }
+        }
+    }
+    Ok(())
+}
+
+impl<'a> TtIter<'a> {
+    fn eat_separator(&mut self, separator: &Separator) -> bool {
+        let mut fork = self.clone();
+        let ok = match separator {
+            Separator::Ident(lhs) => match fork.expect_ident() {
+                Ok(rhs) => rhs.text == lhs.text,
+                _ => false,
+            },
+            Separator::Literal(lhs) => match fork.expect_literal() {
+                Ok(rhs) => match rhs {
+                    tt::Leaf::Literal(rhs) => rhs.text == lhs.text,
+                    tt::Leaf::Ident(rhs) => rhs.text == lhs.text,
+                    tt::Leaf::Punct(_) => false,
+                },
+                _ => false,
+            },
+            Separator::Puncts(lhss) => lhss.iter().all(|lhs| match fork.expect_punct() {
+                Ok(rhs) => rhs.char == lhs.char,
+                _ => false,
+            }),
+        };
+        if ok {
+            *self = fork;
+        }
+        ok
+    }
+
+    pub(crate) fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> {
+        match self.peek_n(0) {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => {
+                return self.expect_lifetime();
+            }
+            _ => (),
+        }
+
+        let tt = self.next().ok_or_else(|| ())?.clone();
+        let punct = match tt {
+            tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => {
+                punct
+            }
+            _ => return Ok(tt),
+        };
+
+        let (second, third) = match (self.peek_n(0), self.peek_n(1)) {
+            (
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))),
+                Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))),
+            ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)),
+            (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None),
+            _ => return Ok(tt),
+        };
+
+        match (punct.char, second, third) {
+            ('.', '.', Some('.'))
+            | ('.', '.', Some('='))
+            | ('<', '<', Some('='))
+            | ('>', '>', Some('=')) => {
+                let tt2 = self.next().unwrap().clone();
+                let tt3 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into())
+            }
+            ('-', '=', None)
+            | ('-', '>', None)
+            | (':', ':', None)
+            | ('!', '=', None)
+            | ('.', '.', None)
+            | ('*', '=', None)
+            | ('/', '=', None)
+            | ('&', '&', None)
+            | ('&', '=', None)
+            | ('%', '=', None)
+            | ('^', '=', None)
+            | ('+', '=', None)
+            | ('<', '<', None)
+            | ('<', '=', None)
+            | ('=', '=', None)
+            | ('=', '>', None)
+            | ('>', '=', None)
+            | ('>', '>', None)
+            | ('|', '=', None)
+            | ('|', '|', None) => {
+                let tt2 = self.next().unwrap().clone();
+                Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into())
+            }
+            _ => Ok(tt),
+        }
+    }
+
+    pub(crate) fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> {
+        let punct = self.expect_punct()?;
+        if punct.char != '\'' {
+            return Err(());
+        }
+        let ident = self.expect_ident()?;
+
+        Ok(tt::Subtree {
+            delimiter: None,
+            token_trees: vec![
+                tt::Leaf::Punct(*punct).into(),
+                tt::Leaf::Ident(ident.clone()).into(),
+            ],
+        }
+        .into())
+    }
+
+    pub(crate) fn expect_fragment(
+        &mut self,
+        fragment_kind: parser::FragmentKind,
+    ) -> ExpandResult<Option<tt::TokenTree>> {
+        pub(crate) struct OffsetTokenSink<'a> {
+            pub(crate) cursor: Cursor<'a>,
+            pub(crate) error: bool,
+        }
+
+        impl<'a> TreeSink for OffsetTokenSink<'a> {
+            fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+                if kind == SyntaxKind::LIFETIME {
+                    n_tokens = 2;
+                }
+                for _ in 0..n_tokens {
+                    self.cursor = self.cursor.bump_subtree();
+                }
+            }
+            fn start_node(&mut self, _kind: SyntaxKind) {}
+            fn finish_node(&mut self) {}
+            fn error(&mut self, _error: parser::ParseError) {
+                self.error = true;
+            }
+        }
+
+        let buffer = TokenBuffer::new(&self.inner.as_slice());
+        let mut src = SubtreeTokenSource::new(&buffer);
+        let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
+
+        parser::parse_fragment(&mut src, &mut sink, fragment_kind);
+
+        let mut err = None;
+        if !sink.cursor.is_root() || sink.error {
+            err = Some(err!("expected {:?}", fragment_kind));
+        }
+
+        let mut curr = buffer.begin();
+        let mut res = vec![];
+
+        if sink.cursor.is_root() {
+            while curr != sink.cursor {
+                if let Some(token) = curr.token_tree() {
+                    res.push(token);
+                }
+                curr = curr.bump();
+            }
+        }
+        self.inner = self.inner.as_slice()[res.len()..].iter();
+        if res.len() == 0 && err.is_none() {
+            err = Some(err!("no tokens consumed"));
+        }
+        let res = match res.len() {
+            1 => Some(res[0].clone()),
+            0 => None,
+            _ => Some(tt::TokenTree::Subtree(tt::Subtree {
+                delimiter: None,
+                token_trees: res.into_iter().cloned().collect(),
+            })),
+        };
+        ExpandResult(res, err)
+    }
+
+    pub(crate) fn eat_vis(&mut self) -> Option<tt::TokenTree> {
+        let mut fork = self.clone();
+        match fork.expect_fragment(Visibility) {
+            ExpandResult(tt, None) => {
+                *self = fork;
+                tt
+            }
+            ExpandResult(_, Some(_)) => None,
+        }
+    }
+}
+
+pub(super) fn match_repeat(
+    res: &mut Match,
+    pattern: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+    src: &mut TtIter,
+) -> Result<(), ExpandError> {
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a propper macro-by-example implementation
+    let mut limit = 65536;
+    let mut counter = 0;
+
+    for i in 0.. {
+        let mut fork = src.clone();
+
+        if let Some(separator) = &separator {
+            if i != 0 && !fork.eat_separator(separator) {
+                break;
+            }
+        }
+
+        let mut nested = Match::default();
+        match_subtree(&mut nested, pattern, &mut fork)?;
+        if nested.err.is_none() {
+            limit -= 1;
+            if limit == 0 {
+                log::warn!(
+                    "match_lhs exceeded repeat pattern limit => {:#?}\n{:#?}\n{:#?}\n{:#?}",
+                    pattern,
+                    src,
+                    kind,
+                    separator
+                );
+                break;
+            }
+            *src = fork;
+
+            if let Err(err) = res.bindings.push_nested(counter, nested.bindings) {
+                res.add_err(err);
+            }
+            counter += 1;
+            if counter == 1 {
+                if let RepeatKind::ZeroOrOne = kind {
+                    break;
+                }
+            }
+        } else {
+            break;
+        }
+    }
+
+    match (kind, counter) {
+        (RepeatKind::OneOrMore, 0) => {
+            res.add_err(ExpandError::UnexpectedToken);
+        }
+        (_, 0) => {
+            // Collect all empty variables in subtrees
+            let mut vars = Vec::new();
+            collect_vars(&mut vars, pattern)?;
+            for var in vars {
+                res.bindings.push_empty(&var)
+            }
+        }
+        _ => (),
+    }
+    Ok(())
+}
+
+fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragment>> {
+    let fragment = match kind {
+        "path" => Path,
+        "expr" => Expr,
+        "ty" => Type,
+        "pat" => Pattern,
+        "stmt" => Statement,
+        "block" => Block,
+        "meta" => MetaItem,
+        "item" => Item,
+        _ => {
+            let tt_result = match kind {
+                "ident" => input
+                    .expect_ident()
+                    .map(|ident| Some(tt::Leaf::from(ident.clone()).into()))
+                    .map_err(|()| err!("expected ident")),
+                "tt" => input.expect_tt().map(Some).map_err(|()| err!()),
+                "lifetime" => input
+                    .expect_lifetime()
+                    .map(|tt| Some(tt))
+                    .map_err(|()| err!("expected lifetime")),
+                "literal" => input
+                    .expect_literal()
+                    .map(|literal| Some(tt::Leaf::from(literal.clone()).into()))
+                    .map_err(|()| err!()),
+                // `vis` is optional
+                "vis" => match input.eat_vis() {
+                    Some(vis) => Ok(Some(vis)),
+                    None => Ok(None),
+                },
+                _ => Err(ExpandError::UnexpectedToken),
+            };
+            return tt_result.map(|it| it.map(Fragment::Tokens)).into();
+        }
+    };
+    let result = input.expect_fragment(fragment);
+    result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) })
+}
+
+fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &tt::Subtree) -> Result<(), ExpandError> {
+    for op in parse_pattern(pattern) {
+        match op? {
+            Op::Var { name, .. } => buf.push(name.clone()),
+            Op::TokenTree(tt::TokenTree::Leaf(_)) => (),
+            Op::TokenTree(tt::TokenTree::Subtree(subtree)) => collect_vars(buf, subtree)?,
+            Op::Repeat { subtree, .. } => collect_vars(buf, subtree)?,
+        }
+    }
+    Ok(())
+}

crates/mbe/src/mbe_expander/transcriber.rs

@@ -0,0 +1,254 @@
+//! Transcriber takes a template, like `fn $ident() {}`, a set of bindings like
+//! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}`
+
+use syntax::SmolStr;
+
+use super::ExpandResult;
+use crate::{
+    mbe_expander::{Binding, Bindings, Fragment},
+    parser::{parse_template, Op, RepeatKind, Separator},
+    ExpandError,
+};
+
+impl Bindings {
+    fn contains(&self, name: &str) -> bool {
+        self.inner.contains_key(name)
+    }
+
+    fn get(&self, name: &str, nesting: &mut [NestingState]) -> Result<&Fragment, ExpandError> {
+        let mut b = self.inner.get(name).ok_or_else(|| {
+            ExpandError::BindingError(format!("could not find binding `{}`", name))
+        })?;
+        for nesting_state in nesting.iter_mut() {
+            nesting_state.hit = true;
+            b = match b {
+                Binding::Fragment(_) => break,
+                Binding::Nested(bs) => bs.get(nesting_state.idx).ok_or_else(|| {
+                    nesting_state.at_end = true;
+                    ExpandError::BindingError(format!("could not find nested binding `{}`", name))
+                })?,
+                Binding::Empty => {
+                    nesting_state.at_end = true;
+                    return Err(ExpandError::BindingError(format!(
+                        "could not find empty binding `{}`",
+                        name
+                    )));
+                }
+            };
+        }
+        match b {
+            Binding::Fragment(it) => Ok(it),
+            Binding::Nested(_) => Err(ExpandError::BindingError(format!(
+                "expected simple binding, found nested binding `{}`",
+                name
+            ))),
+            Binding::Empty => Err(ExpandError::BindingError(format!(
+                "expected simple binding, found empty binding `{}`",
+                name
+            ))),
+        }
+    }
+}
+
+pub(super) fn transcribe(template: &tt::Subtree, bindings: &Bindings) -> ExpandResult<tt::Subtree> {
+    assert!(template.delimiter == None);
+    let mut ctx = ExpandCtx { bindings: &bindings, nesting: Vec::new() };
+    let mut arena: Vec<tt::TokenTree> = Vec::new();
+    expand_subtree(&mut ctx, template, &mut arena)
+}
+
+#[derive(Debug)]
+struct NestingState {
+    idx: usize,
+    /// `hit` is currently necessary to tell `expand_repeat` if it should stop
+    /// because there is no variable in use by the current repetition
+    hit: bool,
+    /// `at_end` is currently necessary to tell `expand_repeat` if it should stop
+    /// because there is no more value avaible for the current repetition
+    at_end: bool,
+}
+
+#[derive(Debug)]
+struct ExpandCtx<'a> {
+    bindings: &'a Bindings,
+    nesting: Vec<NestingState>,
+}
+
+fn expand_subtree(
+    ctx: &mut ExpandCtx,
+    template: &tt::Subtree,
+    arena: &mut Vec<tt::TokenTree>,
+) -> ExpandResult<tt::Subtree> {
+    // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation
+    let start_elements = arena.len();
+    let mut err = None;
+    for op in parse_template(template) {
+        let op = match op {
+            Ok(op) => op,
+            Err(e) => {
+                err = Some(e);
+                break;
+            }
+        };
+        match op {
+            Op::TokenTree(tt @ tt::TokenTree::Leaf(..)) => arena.push(tt.clone()),
+            Op::TokenTree(tt::TokenTree::Subtree(tt)) => {
+                let ExpandResult(tt, e) = expand_subtree(ctx, tt, arena);
+                err = err.or(e);
+                arena.push(tt.into());
+            }
+            Op::Var { name, kind: _ } => {
+                let ExpandResult(fragment, e) = expand_var(ctx, name);
+                err = err.or(e);
+                push_fragment(arena, fragment);
+            }
+            Op::Repeat { subtree, kind, separator } => {
+                let ExpandResult(fragment, e) = expand_repeat(ctx, subtree, kind, separator, arena);
+                err = err.or(e);
+                push_fragment(arena, fragment)
+            }
+        }
+    }
+    // drain the elements added in this instance of expand_subtree
+    let tts = arena.drain(start_elements..arena.len()).collect();
+    ExpandResult(tt::Subtree { delimiter: template.delimiter, token_trees: tts }, err)
+}
+
+fn expand_var(ctx: &mut ExpandCtx, v: &SmolStr) -> ExpandResult<Fragment> {
+    if v == "crate" {
+        // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.
+        let tt =
+            tt::Leaf::from(tt::Ident { text: "$crate".into(), id: tt::TokenId::unspecified() })
+                .into();
+        ExpandResult::ok(Fragment::Tokens(tt))
+    } else if !ctx.bindings.contains(v) {
+        // Note that it is possible to have a `$var` inside a macro which is not bound.
+        // For example:
+        // ```
+        // macro_rules! foo {
+        //     ($a:ident, $b:ident, $c:tt) => {
+        //         macro_rules! bar {
+        //             ($bi:ident) => {
+        //                 fn $bi() -> u8 {$c}
+        //             }
+        //         }
+        //     }
+        // ```
+        // We just treat it a normal tokens
+        let tt = tt::Subtree {
+            delimiter: None,
+            token_trees: vec![
+                tt::Leaf::from(tt::Punct {
+                    char: '$',
+                    spacing: tt::Spacing::Alone,
+                    id: tt::TokenId::unspecified(),
+                })
+                .into(),
+                tt::Leaf::from(tt::Ident { text: v.clone(), id: tt::TokenId::unspecified() })
+                    .into(),
+            ],
+        }
+        .into();
+        ExpandResult::ok(Fragment::Tokens(tt))
+    } else {
+        ctx.bindings.get(&v, &mut ctx.nesting).map_or_else(
+            |e| ExpandResult(Fragment::Tokens(tt::TokenTree::empty()), Some(e)),
+            |b| ExpandResult::ok(b.clone()),
+        )
+    }
+}
+
+fn expand_repeat(
+    ctx: &mut ExpandCtx,
+    template: &tt::Subtree,
+    kind: RepeatKind,
+    separator: Option<Separator>,
+    arena: &mut Vec<tt::TokenTree>,
+) -> ExpandResult<Fragment> {
+    let mut buf: Vec<tt::TokenTree> = Vec::new();
+    ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false });
+    // Dirty hack to make macro-expansion terminate.
+    // This should be replaced by a proper macro-by-example implementation
+    let limit = 65536;
+    let mut has_seps = 0;
+    let mut counter = 0;
+
+    loop {
+        let ExpandResult(mut t, e) = expand_subtree(ctx, template, arena);
+        let nesting_state = ctx.nesting.last_mut().unwrap();
+        if nesting_state.at_end || !nesting_state.hit {
+            break;
+        }
+        nesting_state.idx += 1;
+        nesting_state.hit = false;
+
+        counter += 1;
+        if counter == limit {
+            log::warn!(
+                "expand_tt excced in repeat pattern exceed limit => {:#?}\n{:#?}",
+                template,
+                ctx
+            );
+            break;
+        }
+
+        if e.is_some() {
+            continue;
+        }
+
+        t.delimiter = None;
+        push_subtree(&mut buf, t);
+
+        if let Some(ref sep) = separator {
+            match sep {
+                Separator::Ident(ident) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(ident.clone()).into());
+                }
+                Separator::Literal(lit) => {
+                    has_seps = 1;
+                    buf.push(tt::Leaf::from(lit.clone()).into());
+                }
+
+                Separator::Puncts(puncts) => {
+                    has_seps = puncts.len();
+                    for punct in puncts {
+                        buf.push(tt::Leaf::from(*punct).into());
+                    }
+                }
+            }
+        }
+
+        if RepeatKind::ZeroOrOne == kind {
+            break;
+        }
+    }
+
+    ctx.nesting.pop().unwrap();
+    for _ in 0..has_seps {
+        buf.pop();
+    }
+
+    // Check if it is a single token subtree without any delimiter
+    // e.g {Delimiter:None> ['>'] /Delimiter:None>}
+    let tt = tt::Subtree { delimiter: None, token_trees: buf }.into();
+
+    if RepeatKind::OneOrMore == kind && counter == 0 {
+        return ExpandResult(Fragment::Tokens(tt), Some(ExpandError::UnexpectedToken));
+    }
+    ExpandResult::ok(Fragment::Tokens(tt))
+}
+
+fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) {
+    match fragment {
+        Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt),
+        Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt),
+    }
+}
+
+fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) {
+    match tt.delimiter {
+        None => buf.extend(tt.token_trees),
+        _ => buf.push(tt.into()),
+    }
+}

crates/mbe/src/parser.rs

@@ -0,0 +1,184 @@
+//! Parser recognizes special macro syntax, `$var` and `$(repeat)*`, in token
+//! trees.
+
+use smallvec::SmallVec;
+use syntax::SmolStr;
+
+use crate::{tt_iter::TtIter, ExpandError};
+
+#[derive(Debug)]
+pub(crate) enum Op<'a> {
+    Var { name: &'a SmolStr, kind: Option<&'a SmolStr> },
+    Repeat { subtree: &'a tt::Subtree, kind: RepeatKind, separator: Option<Separator> },
+    TokenTree(&'a tt::TokenTree),
+}
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(crate) enum RepeatKind {
+    ZeroOrMore,
+    OneOrMore,
+    ZeroOrOne,
+}
+
+#[derive(Clone, Debug, Eq)]
+pub(crate) enum Separator {
+    Literal(tt::Literal),
+    Ident(tt::Ident),
+    Puncts(SmallVec<[tt::Punct; 3]>),
+}
+
+// Note that when we compare a Separator, we just care about its textual value.
+impl PartialEq for Separator {
+    fn eq(&self, other: &Separator) -> bool {
+        use Separator::*;
+
+        match (self, other) {
+            (Ident(ref a), Ident(ref b)) => a.text == b.text,
+            (Literal(ref a), Literal(ref b)) => a.text == b.text,
+            (Puncts(ref a), Puncts(ref b)) if a.len() == b.len() => {
+                let a_iter = a.iter().map(|a| a.char);
+                let b_iter = b.iter().map(|b| b.char);
+                a_iter.eq(b_iter)
+            }
+            _ => false,
+        }
+    }
+}
+
+pub(crate) fn parse_template(
+    template: &tt::Subtree,
+) -> impl Iterator<Item = Result<Op<'_>, ExpandError>> {
+    parse_inner(template, Mode::Template)
+}
+
+pub(crate) fn parse_pattern(
+    pattern: &tt::Subtree,
+) -> impl Iterator<Item = Result<Op<'_>, ExpandError>> {
+    parse_inner(pattern, Mode::Pattern)
+}
+
+#[derive(Clone, Copy)]
+enum Mode {
+    Pattern,
+    Template,
+}
+
+fn parse_inner(src: &tt::Subtree, mode: Mode) -> impl Iterator<Item = Result<Op<'_>, ExpandError>> {
+    let mut src = TtIter::new(src);
+    std::iter::from_fn(move || {
+        let first = src.next()?;
+        Some(next_op(first, &mut src, mode))
+    })
+}
+
+macro_rules! err {
+    ($($tt:tt)*) => {
+        ExpandError::UnexpectedToken
+    };
+}
+
+macro_rules! bail {
+    ($($tt:tt)*) => {
+        return Err(err!($($tt)*))
+    };
+}
+
+fn next_op<'a>(
+    first: &'a tt::TokenTree,
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Op<'a>, ExpandError> {
+    let res = match first {
+        tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. })) => {
+            // Note that the '$' itself is a valid token inside macro_rules.
+            let second = match src.next() {
+                None => return Ok(Op::TokenTree(first)),
+                Some(it) => it,
+            };
+            match second {
+                tt::TokenTree::Subtree(subtree) => {
+                    let (separator, kind) = parse_repeat(src)?;
+                    Op::Repeat { subtree, separator, kind }
+                }
+                tt::TokenTree::Leaf(leaf) => match leaf {
+                    tt::Leaf::Punct(..) => return Err(ExpandError::UnexpectedToken),
+                    tt::Leaf::Ident(ident) => {
+                        let name = &ident.text;
+                        let kind = eat_fragment_kind(src, mode)?;
+                        Op::Var { name, kind }
+                    }
+                    tt::Leaf::Literal(lit) => {
+                        if is_boolean_literal(lit) {
+                            let name = &lit.text;
+                            let kind = eat_fragment_kind(src, mode)?;
+                            Op::Var { name, kind }
+                        } else {
+                            bail!("bad var 2");
+                        }
+                    }
+                },
+            }
+        }
+        tt => Op::TokenTree(tt),
+    };
+    Ok(res)
+}
+
+fn eat_fragment_kind<'a>(
+    src: &mut TtIter<'a>,
+    mode: Mode,
+) -> Result<Option<&'a SmolStr>, ExpandError> {
+    if let Mode::Pattern = mode {
+        src.expect_char(':').map_err(|()| err!("bad fragment specifier 1"))?;
+        let ident = src.expect_ident().map_err(|()| err!("bad fragment specifier 1"))?;
+        return Ok(Some(&ident.text));
+    };
+    Ok(None)
+}
+
+fn is_boolean_literal(lit: &tt::Literal) -> bool {
+    matches!(lit.text.as_str(), "true" | "false")
+}
+
+fn parse_repeat(src: &mut TtIter) -> Result<(Option<Separator>, RepeatKind), ExpandError> {
+    let mut separator = Separator::Puncts(SmallVec::new());
+    for tt in src {
+        let tt = match tt {
+            tt::TokenTree::Leaf(leaf) => leaf,
+            tt::TokenTree::Subtree(_) => return Err(ExpandError::InvalidRepeat),
+        };
+        let has_sep = match &separator {
+            Separator::Puncts(puncts) => !puncts.is_empty(),
+            _ => true,
+        };
+        match tt {
+            tt::Leaf::Ident(_) | tt::Leaf::Literal(_) if has_sep => {
+                return Err(ExpandError::InvalidRepeat)
+            }
+            tt::Leaf::Ident(ident) => separator = Separator::Ident(ident.clone()),
+            tt::Leaf::Literal(lit) => separator = Separator::Literal(lit.clone()),
+            tt::Leaf::Punct(punct) => {
+                let repeat_kind = match punct.char {
+                    '*' => RepeatKind::ZeroOrMore,
+                    '+' => RepeatKind::OneOrMore,
+                    '?' => RepeatKind::ZeroOrOne,
+                    _ => {
+                        match &mut separator {
+                            Separator::Puncts(puncts) => {
+                                if puncts.len() == 3 {
+                                    return Err(ExpandError::InvalidRepeat);
+                                }
+                                puncts.push(punct.clone())
+                            }
+                            _ => return Err(ExpandError::InvalidRepeat),
+                        }
+                        continue;
+                    }
+                };
+                let separator = if has_sep { Some(separator) } else { None };
+                return Ok((separator, repeat_kind));
+            }
+        }
+    }
+    Err(ExpandError::InvalidRepeat)
+}

crates/mbe/src/subtree_source.rs

@@ -0,0 +1,197 @@
+//! FIXME: write short doc here
+
+use parser::{Token, TokenSource};
+use std::cell::{Cell, Ref, RefCell};
+use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T};
+use tt::buffer::{Cursor, TokenBuffer};
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+struct TtToken {
+    pub kind: SyntaxKind,
+    pub is_joint_to_next: bool,
+    pub text: SmolStr,
+}
+
+pub(crate) struct SubtreeTokenSource<'a> {
+    cached_cursor: Cell<Cursor<'a>>,
+    cached: RefCell<Vec<Option<TtToken>>>,
+    curr: (Token, usize),
+}
+
+impl<'a> SubtreeTokenSource<'a> {
+    // Helper function used in test
+    #[cfg(test)]
+    pub fn text(&self) -> SmolStr {
+        match *self.get(self.curr.1) {
+            Some(ref tt) => tt.text.clone(),
+            _ => SmolStr::new(""),
+        }
+    }
+}
+
+impl<'a> SubtreeTokenSource<'a> {
+    pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
+        let cursor = buffer.begin();
+
+        let mut res = SubtreeTokenSource {
+            curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
+            cached_cursor: Cell::new(cursor),
+            cached: RefCell::new(Vec::with_capacity(10)),
+        };
+        res.curr = (res.mk_token(0), 0);
+        res
+    }
+
+    fn mk_token(&self, pos: usize) -> Token {
+        match *self.get(pos) {
+            Some(ref tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
+            None => Token { kind: EOF, is_jointed_to_next: false },
+        }
+    }
+
+    fn get(&self, pos: usize) -> Ref<Option<TtToken>> {
+        fn is_lifetime(c: Cursor) -> Option<(Cursor, SmolStr)> {
+            let tkn = c.token_tree();
+
+            if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tkn {
+                if punct.char == '\'' {
+                    let next = c.bump();
+                    if let Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) = next.token_tree() {
+                        let res_cursor = next.bump();
+                        let text = SmolStr::new("'".to_string() + &ident.to_string());
+
+                        return Some((res_cursor, text));
+                    } else {
+                        panic!("Next token must be ident : {:#?}", next.token_tree());
+                    }
+                }
+            }
+
+            None
+        }
+
+        if pos < self.cached.borrow().len() {
+            return Ref::map(self.cached.borrow(), |c| &c[pos]);
+        }
+
+        {
+            let mut cached = self.cached.borrow_mut();
+            while pos >= cached.len() {
+                let cursor = self.cached_cursor.get();
+                if cursor.eof() {
+                    cached.push(None);
+                    continue;
+                }
+
+                if let Some((curr, text)) = is_lifetime(cursor) {
+                    cached.push(Some(TtToken { kind: LIFETIME, is_joint_to_next: false, text }));
+                    self.cached_cursor.set(curr);
+                    continue;
+                }
+
+                match cursor.token_tree() {
+                    Some(tt::TokenTree::Leaf(leaf)) => {
+                        cached.push(Some(convert_leaf(&leaf)));
+                        self.cached_cursor.set(cursor.bump());
+                    }
+                    Some(tt::TokenTree::Subtree(subtree)) => {
+                        self.cached_cursor.set(cursor.subtree().unwrap());
+                        cached.push(Some(convert_delim(subtree.delimiter_kind(), false)));
+                    }
+                    None => {
+                        if let Some(subtree) = cursor.end() {
+                            cached.push(Some(convert_delim(subtree.delimiter_kind(), true)));
+                            self.cached_cursor.set(cursor.bump());
+                        }
+                    }
+                }
+            }
+        }
+
+        Ref::map(self.cached.borrow(), |c| &c[pos])
+    }
+}
+
+impl<'a> TokenSource for SubtreeTokenSource<'a> {
+    fn current(&self) -> Token {
+        self.curr.0
+    }
+
+    /// Lookahead n token
+    fn lookahead_nth(&self, n: usize) -> Token {
+        self.mk_token(self.curr.1 + n)
+    }
+
+    /// bump cursor to next token
+    fn bump(&mut self) {
+        if self.current().kind == EOF {
+            return;
+        }
+
+        self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1);
+    }
+
+    /// Is the current token a specified keyword?
+    fn is_keyword(&self, kw: &str) -> bool {
+        match *self.get(self.curr.1) {
+            Some(ref t) => t.text == *kw,
+            _ => false,
+        }
+    }
+}
+
+fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
+    let (kinds, texts) = match d {
+        Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"),
+        Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"),
+        Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"),
+        None => ([L_DOLLAR, R_DOLLAR], ""),
+    };
+
+    let idx = closing as usize;
+    let kind = kinds[idx];
+    let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" };
+    TtToken { kind, is_joint_to_next: false, text: SmolStr::new(text) }
+}
+
+fn convert_literal(l: &tt::Literal) -> TtToken {
+    let kind = lex_single_syntax_kind(&l.text)
+        .map(|(kind, _error)| kind)
+        .filter(|kind| kind.is_literal())
+        .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l));
+
+    TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
+}
+
+fn convert_ident(ident: &tt::Ident) -> TtToken {
+    let kind = match ident.text.as_ref() {
+        "true" => T![true],
+        "false" => T![false],
+        i if i.starts_with('\'') => LIFETIME,
+        _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
+    };
+
+    TtToken { kind, is_joint_to_next: false, text: ident.text.clone() }
+}
+
+fn convert_punct(p: tt::Punct) -> TtToken {
+    let kind = match SyntaxKind::from_char(p.char) {
+        None => panic!("{:#?} is not a valid punct", p),
+        Some(kind) => kind,
+    };
+
+    let text = {
+        let mut buf = [0u8; 4];
+        let s: &str = p.char.encode_utf8(&mut buf);
+        SmolStr::new(s)
+    };
+    TtToken { kind, is_joint_to_next: p.spacing == tt::Spacing::Joint, text }
+}
+
+fn convert_leaf(leaf: &tt::Leaf) -> TtToken {
+    match leaf {
+        tt::Leaf::Literal(l) => convert_literal(l),
+        tt::Leaf::Ident(ident) => convert_ident(ident),
+        tt::Leaf::Punct(punct) => convert_punct(*punct),
+    }
+}

crates/mbe/src/syntax_bridge.rs

@@ -0,0 +1,832 @@
+//! FIXME: write short doc here
+
+use parser::{FragmentKind, ParseError, TreeSink};
+use rustc_hash::FxHashMap;
+use syntax::{
+    ast::{self, make::tokens::doc_comment},
+    tokenize, AstToken, Parse, SmolStr, SyntaxKind,
+    SyntaxKind::*,
+    SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, Token as RawToken, T,
+};
+use tt::buffer::{Cursor, TokenBuffer};
+
+use crate::subtree_source::SubtreeTokenSource;
+use crate::ExpandError;
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
+pub enum TokenTextRange {
+    Token(TextRange),
+    Delimiter(TextRange, TextRange),
+}
+
+impl TokenTextRange {
+    pub fn by_kind(self, kind: SyntaxKind) -> Option<TextRange> {
+        match self {
+            TokenTextRange::Token(it) => Some(it),
+            TokenTextRange::Delimiter(open, close) => match kind {
+                T!['{'] | T!['('] | T!['['] => Some(open),
+                T!['}'] | T![')'] | T![']'] => Some(close),
+                _ => None,
+            },
+        }
+    }
+}
+
+/// Maps `tt::TokenId` to the relative range of the original token.
+#[derive(Debug, PartialEq, Eq, Clone, Default)]
+pub struct TokenMap {
+    /// Maps `tt::TokenId` to the *relative* source range.
+    entries: Vec<(tt::TokenId, TokenTextRange)>,
+}
+
+/// Convert the syntax tree (what user has written) to a `TokenTree` (what macro
+/// will consume).
+pub fn ast_to_token_tree(ast: &impl ast::AstNode) -> Option<(tt::Subtree, TokenMap)> {
+    syntax_node_to_token_tree(ast.syntax())
+}
+
+/// Convert the syntax node to a `TokenTree` (what macro
+/// will consume).
+pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, TokenMap)> {
+    let global_offset = node.text_range().start();
+    let mut c = Convertor::new(node, global_offset);
+    let subtree = c.go()?;
+    Some((subtree, c.id_alloc.map))
+}
+
+// The following items are what `rustc` macro can be parsed into :
+// link: https://github.com/rust-lang/rust/blob/9ebf47851a357faa4cd97f4b1dc7835f6376e639/src/libsyntax/ext/expand.rs#L141
+// * Expr(P<ast::Expr>)                     -> token_tree_to_expr
+// * Pat(P<ast::Pat>)                       -> token_tree_to_pat
+// * Ty(P<ast::Ty>)                         -> token_tree_to_ty
+// * Stmts(SmallVec<[ast::Stmt; 1]>)        -> token_tree_to_stmts
+// * Items(SmallVec<[P<ast::Item>; 1]>)     -> token_tree_to_items
+//
+// * TraitItems(SmallVec<[ast::TraitItem; 1]>)
+// * AssocItems(SmallVec<[ast::AssocItem; 1]>)
+// * ForeignItems(SmallVec<[ast::ForeignItem; 1]>
+
+pub fn token_tree_to_syntax_node(
+    tt: &tt::Subtree,
+    fragment_kind: FragmentKind,
+) -> Result<(Parse<SyntaxNode>, TokenMap), ExpandError> {
+    let tmp;
+    let tokens = match tt {
+        tt::Subtree { delimiter: None, token_trees } => token_trees.as_slice(),
+        _ => {
+            tmp = [tt.clone().into()];
+            &tmp[..]
+        }
+    };
+    let buffer = TokenBuffer::new(&tokens);
+    let mut token_source = SubtreeTokenSource::new(&buffer);
+    let mut tree_sink = TtTreeSink::new(buffer.begin());
+    parser::parse_fragment(&mut token_source, &mut tree_sink, fragment_kind);
+    if tree_sink.roots.len() != 1 {
+        return Err(ExpandError::ConversionError);
+    }
+    //FIXME: would be cool to report errors
+    let (parse, range_map) = tree_sink.finish();
+    Ok((parse, range_map))
+}
+
+/// Convert a string to a `TokenTree`
+pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> {
+    let (tokens, errors) = tokenize(text);
+    if !errors.is_empty() {
+        return None;
+    }
+
+    let mut conv = RawConvertor {
+        text,
+        offset: TextSize::default(),
+        inner: tokens.iter(),
+        id_alloc: TokenIdAlloc {
+            map: Default::default(),
+            global_offset: TextSize::default(),
+            next_id: 0,
+        },
+    };
+
+    let subtree = conv.go()?;
+    Some((subtree, conv.id_alloc.map))
+}
+
+impl TokenMap {
+    pub fn token_by_range(&self, relative_range: TextRange) -> Option<tt::TokenId> {
+        let &(token_id, _) = self.entries.iter().find(|(_, range)| match range {
+            TokenTextRange::Token(it) => *it == relative_range,
+            TokenTextRange::Delimiter(open, close) => {
+                *open == relative_range || *close == relative_range
+            }
+        })?;
+        Some(token_id)
+    }
+
+    pub fn range_by_token(&self, token_id: tt::TokenId) -> Option<TokenTextRange> {
+        let &(_, range) = self.entries.iter().find(|(tid, _)| *tid == token_id)?;
+        Some(range)
+    }
+
+    fn insert(&mut self, token_id: tt::TokenId, relative_range: TextRange) {
+        self.entries.push((token_id, TokenTextRange::Token(relative_range)));
+    }
+
+    fn insert_delim(
+        &mut self,
+        token_id: tt::TokenId,
+        open_relative_range: TextRange,
+        close_relative_range: TextRange,
+    ) -> usize {
+        let res = self.entries.len();
+        self.entries
+            .push((token_id, TokenTextRange::Delimiter(open_relative_range, close_relative_range)));
+        res
+    }
+
+    fn update_close_delim(&mut self, idx: usize, close_relative_range: TextRange) {
+        let (_, token_text_range) = &mut self.entries[idx];
+        if let TokenTextRange::Delimiter(dim, _) = token_text_range {
+            *token_text_range = TokenTextRange::Delimiter(*dim, close_relative_range);
+        }
+    }
+
+    fn remove_delim(&mut self, idx: usize) {
+        // FIXME: This could be accidently quadratic
+        self.entries.remove(idx);
+    }
+}
+
+/// Returns the textual content of a doc comment block as a quoted string
+/// That is, strips leading `///` (or `/**`, etc)
+/// and strips the ending `*/`
+/// And then quote the string, which is needed to convert to `tt::Literal`
+fn doc_comment_text(comment: &ast::Comment) -> SmolStr {
+    let prefix_len = comment.prefix().len();
+    let mut text = &comment.text()[prefix_len..];
+
+    // Remove ending "*/"
+    if comment.kind().shape == ast::CommentShape::Block {
+        text = &text[0..text.len() - 2];
+    }
+
+    // Quote the string
+    // Note that `tt::Literal` expect an escaped string
+    let text = format!("{:?}", text.escape_default().to_string());
+    text.into()
+}
+
+fn convert_doc_comment(token: &syntax::SyntaxToken) -> Option<Vec<tt::TokenTree>> {
+    let comment = ast::Comment::cast(token.clone())?;
+    let doc = comment.kind().doc?;
+
+    // Make `doc="\" Comments\""
+    let mut meta_tkns = Vec::new();
+    meta_tkns.push(mk_ident("doc"));
+    meta_tkns.push(mk_punct('='));
+    meta_tkns.push(mk_doc_literal(&comment));
+
+    // Make `#![]`
+    let mut token_trees = Vec::new();
+    token_trees.push(mk_punct('#'));
+    if let ast::CommentPlacement::Inner = doc {
+        token_trees.push(mk_punct('!'));
+    }
+    token_trees.push(tt::TokenTree::from(tt::Subtree {
+        delimiter: Some(tt::Delimiter {
+            kind: tt::DelimiterKind::Bracket,
+            id: tt::TokenId::unspecified(),
+        }),
+        token_trees: meta_tkns,
+    }));
+
+    return Some(token_trees);
+
+    // Helper functions
+    fn mk_ident(s: &str) -> tt::TokenTree {
+        tt::TokenTree::from(tt::Leaf::from(tt::Ident {
+            text: s.into(),
+            id: tt::TokenId::unspecified(),
+        }))
+    }
+
+    fn mk_punct(c: char) -> tt::TokenTree {
+        tt::TokenTree::from(tt::Leaf::from(tt::Punct {
+            char: c,
+            spacing: tt::Spacing::Alone,
+            id: tt::TokenId::unspecified(),
+        }))
+    }
+
+    fn mk_doc_literal(comment: &ast::Comment) -> tt::TokenTree {
+        let lit = tt::Literal { text: doc_comment_text(comment), id: tt::TokenId::unspecified() };
+
+        tt::TokenTree::from(tt::Leaf::from(lit))
+    }
+}
+
+struct TokenIdAlloc {
+    map: TokenMap,
+    global_offset: TextSize,
+    next_id: u32,
+}
+
+impl TokenIdAlloc {
+    fn alloc(&mut self, absolute_range: TextRange) -> tt::TokenId {
+        let relative_range = absolute_range - self.global_offset;
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+        self.map.insert(token_id, relative_range);
+        token_id
+    }
+
+    fn open_delim(&mut self, open_abs_range: TextRange) -> (tt::TokenId, usize) {
+        let token_id = tt::TokenId(self.next_id);
+        self.next_id += 1;
+        let idx = self.map.insert_delim(
+            token_id,
+            open_abs_range - self.global_offset,
+            open_abs_range - self.global_offset,
+        );
+        (token_id, idx)
+    }
+
+    fn close_delim(&mut self, idx: usize, close_abs_range: Option<TextRange>) {
+        match close_abs_range {
+            None => {
+                self.map.remove_delim(idx);
+            }
+            Some(close) => {
+                self.map.update_close_delim(idx, close - self.global_offset);
+            }
+        }
+    }
+}
+
+/// A Raw Token (straightly from lexer) convertor
+struct RawConvertor<'a> {
+    text: &'a str,
+    offset: TextSize,
+    id_alloc: TokenIdAlloc,
+    inner: std::slice::Iter<'a, RawToken>,
+}
+
+trait SrcToken: std::fmt::Debug {
+    fn kind(&self) -> SyntaxKind;
+
+    fn to_char(&self) -> Option<char>;
+
+    fn to_text(&self) -> SmolStr;
+}
+
+trait TokenConvertor {
+    type Token: SrcToken;
+
+    fn go(&mut self) -> Option<tt::Subtree> {
+        let mut subtree = tt::Subtree::default();
+        subtree.delimiter = None;
+        while self.peek().is_some() {
+            self.collect_leaf(&mut subtree.token_trees);
+        }
+        if subtree.token_trees.is_empty() {
+            return None;
+        }
+        if subtree.token_trees.len() == 1 {
+            if let tt::TokenTree::Subtree(first) = &subtree.token_trees[0] {
+                return Some(first.clone());
+            }
+        }
+        Some(subtree)
+    }
+
+    fn collect_leaf(&mut self, result: &mut Vec<tt::TokenTree>) {
+        let (token, range) = match self.bump() {
+            None => return,
+            Some(it) => it,
+        };
+
+        let k: SyntaxKind = token.kind();
+        if k == COMMENT {
+            if let Some(tokens) = self.convert_doc_comment(&token) {
+                result.extend(tokens);
+            }
+            return;
+        }
+
+        result.push(if k.is_punct() {
+            assert_eq!(range.len(), TextSize::of('.'));
+            let delim = match k {
+                T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
+                T!['{'] => Some((tt::DelimiterKind::Brace, T!['}'])),
+                T!['['] => Some((tt::DelimiterKind::Bracket, T![']'])),
+                _ => None,
+            };
+
+            if let Some((kind, closed)) = delim {
+                let mut subtree = tt::Subtree::default();
+                let (id, idx) = self.id_alloc().open_delim(range);
+                subtree.delimiter = Some(tt::Delimiter { kind, id });
+
+                while self.peek().map(|it| it.kind() != closed).unwrap_or(false) {
+                    self.collect_leaf(&mut subtree.token_trees);
+                }
+                let last_range = match self.bump() {
+                    None => {
+                        // For error resilience, we insert an char punct for the opening delim here
+                        self.id_alloc().close_delim(idx, None);
+                        let leaf: tt::Leaf = tt::Punct {
+                            id: self.id_alloc().alloc(range),
+                            char: token.to_char().unwrap(),
+                            spacing: tt::Spacing::Alone,
+                        }
+                        .into();
+                        result.push(leaf.into());
+                        result.extend(subtree.token_trees);
+                        return;
+                    }
+                    Some(it) => it.1,
+                };
+                self.id_alloc().close_delim(idx, Some(last_range));
+                subtree.into()
+            } else {
+                let spacing = match self.peek() {
+                    Some(next)
+                        if next.kind().is_trivia()
+                            || next.kind() == T!['[']
+                            || next.kind() == T!['{']
+                            || next.kind() == T!['('] =>
+                    {
+                        tt::Spacing::Alone
+                    }
+                    Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
+                    _ => tt::Spacing::Alone,
+                };
+                let char = match token.to_char() {
+                    Some(c) => c,
+                    None => {
+                        panic!("Token from lexer must be single char: token = {:#?}", token);
+                    }
+                };
+                tt::Leaf::from(tt::Punct { char, spacing, id: self.id_alloc().alloc(range) }).into()
+            }
+        } else {
+            macro_rules! make_leaf {
+                ($i:ident) => {
+                    tt::$i { id: self.id_alloc().alloc(range), text: token.to_text() }.into()
+                };
+            }
+            let leaf: tt::Leaf = match k {
+                T![true] | T![false] => make_leaf!(Ident),
+                IDENT => make_leaf!(Ident),
+                k if k.is_keyword() => make_leaf!(Ident),
+                k if k.is_literal() => make_leaf!(Literal),
+                LIFETIME => {
+                    let char_unit = TextSize::of('\'');
+                    let r = TextRange::at(range.start(), char_unit);
+                    let apostrophe = tt::Leaf::from(tt::Punct {
+                        char: '\'',
+                        spacing: tt::Spacing::Joint,
+                        id: self.id_alloc().alloc(r),
+                    });
+                    result.push(apostrophe.into());
+
+                    let r = TextRange::at(range.start() + char_unit, range.len() - char_unit);
+                    let ident = tt::Leaf::from(tt::Ident {
+                        text: SmolStr::new(&token.to_text()[1..]),
+                        id: self.id_alloc().alloc(r),
+                    });
+                    result.push(ident.into());
+                    return;
+                }
+                _ => return,
+            };
+
+            leaf.into()
+        });
+    }
+
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>;
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)>;
+
+    fn peek(&self) -> Option<Self::Token>;
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc;
+}
+
+impl<'a> SrcToken for (RawToken, &'a str) {
+    fn kind(&self) -> SyntaxKind {
+        self.0.kind
+    }
+
+    fn to_char(&self) -> Option<char> {
+        self.1.chars().next()
+    }
+
+    fn to_text(&self) -> SmolStr {
+        self.1.into()
+    }
+}
+
+impl RawConvertor<'_> {}
+
+impl<'a> TokenConvertor for RawConvertor<'a> {
+    type Token = (RawToken, &'a str);
+
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
+        convert_doc_comment(&doc_comment(token.1))
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
+        let token = self.inner.next()?;
+        let range = TextRange::at(self.offset, token.len);
+        self.offset += token.len;
+
+        Some(((*token, &self.text[range]), range))
+    }
+
+    fn peek(&self) -> Option<Self::Token> {
+        let token = self.inner.as_slice().get(0).cloned();
+
+        token.map(|it| {
+            let range = TextRange::at(self.offset, it.len);
+            (it, &self.text[range])
+        })
+    }
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc {
+        &mut self.id_alloc
+    }
+}
+
+struct Convertor {
+    id_alloc: TokenIdAlloc,
+    current: Option<SyntaxToken>,
+    range: TextRange,
+    punct_offset: Option<(SyntaxToken, TextSize)>,
+}
+
+impl Convertor {
+    fn new(node: &SyntaxNode, global_offset: TextSize) -> Convertor {
+        Convertor {
+            id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } },
+            current: node.first_token(),
+            range: node.text_range(),
+            punct_offset: None,
+        }
+    }
+}
+
+#[derive(Debug)]
+enum SynToken {
+    Ordiniary(SyntaxToken),
+    Punch(SyntaxToken, TextSize),
+}
+
+impl SynToken {
+    fn token(&self) -> &SyntaxToken {
+        match self {
+            SynToken::Ordiniary(it) => it,
+            SynToken::Punch(it, _) => it,
+        }
+    }
+}
+
+impl SrcToken for SynToken {
+    fn kind(&self) -> SyntaxKind {
+        self.token().kind()
+    }
+    fn to_char(&self) -> Option<char> {
+        match self {
+            SynToken::Ordiniary(_) => None,
+            SynToken::Punch(it, i) => it.text().chars().nth((*i).into()),
+        }
+    }
+    fn to_text(&self) -> SmolStr {
+        self.token().text().clone()
+    }
+}
+
+impl TokenConvertor for Convertor {
+    type Token = SynToken;
+    fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> {
+        convert_doc_comment(token.token())
+    }
+
+    fn bump(&mut self) -> Option<(Self::Token, TextRange)> {
+        if let Some((punct, offset)) = self.punct_offset.clone() {
+            if usize::from(offset) + 1 < punct.text().len() {
+                let offset = offset + TextSize::of('.');
+                let range = punct.text_range();
+                self.punct_offset = Some((punct.clone(), offset));
+                let range = TextRange::at(range.start() + offset, TextSize::of('.'));
+                return Some((SynToken::Punch(punct, offset), range));
+            }
+        }
+
+        let curr = self.current.clone()?;
+        if !&self.range.contains_range(curr.text_range()) {
+            return None;
+        }
+        self.current = curr.next_token();
+
+        let token = if curr.kind().is_punct() {
+            let range = curr.text_range();
+            let range = TextRange::at(range.start(), TextSize::of('.'));
+            self.punct_offset = Some((curr.clone(), 0.into()));
+            (SynToken::Punch(curr, 0.into()), range)
+        } else {
+            self.punct_offset = None;
+            let range = curr.text_range();
+            (SynToken::Ordiniary(curr), range)
+        };
+
+        Some(token)
+    }
+
+    fn peek(&self) -> Option<Self::Token> {
+        if let Some((punct, mut offset)) = self.punct_offset.clone() {
+            offset = offset + TextSize::of('.');
+            if usize::from(offset) < punct.text().len() {
+                return Some(SynToken::Punch(punct, offset));
+            }
+        }
+
+        let curr = self.current.clone()?;
+        if !self.range.contains_range(curr.text_range()) {
+            return None;
+        }
+
+        let token = if curr.kind().is_punct() {
+            SynToken::Punch(curr, 0.into())
+        } else {
+            SynToken::Ordiniary(curr)
+        };
+        Some(token)
+    }
+
+    fn id_alloc(&mut self) -> &mut TokenIdAlloc {
+        &mut self.id_alloc
+    }
+}
+
+struct TtTreeSink<'a> {
+    buf: String,
+    cursor: Cursor<'a>,
+    open_delims: FxHashMap<tt::TokenId, TextSize>,
+    text_pos: TextSize,
+    inner: SyntaxTreeBuilder,
+    token_map: TokenMap,
+
+    // Number of roots
+    // Use for detect ill-form tree which is not single root
+    roots: smallvec::SmallVec<[usize; 1]>,
+}
+
+impl<'a> TtTreeSink<'a> {
+    fn new(cursor: Cursor<'a>) -> Self {
+        TtTreeSink {
+            buf: String::new(),
+            cursor,
+            open_delims: FxHashMap::default(),
+            text_pos: 0.into(),
+            inner: SyntaxTreeBuilder::default(),
+            roots: smallvec::SmallVec::new(),
+            token_map: TokenMap::default(),
+        }
+    }
+
+    fn finish(self) -> (Parse<SyntaxNode>, TokenMap) {
+        (self.inner.finish(), self.token_map)
+    }
+}
+
+fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> SmolStr {
+    let texts = match d {
+        Some(tt::DelimiterKind::Parenthesis) => "()",
+        Some(tt::DelimiterKind::Brace) => "{}",
+        Some(tt::DelimiterKind::Bracket) => "[]",
+        None => return "".into(),
+    };
+
+    let idx = closing as usize;
+    let text = &texts[idx..texts.len() - (1 - idx)];
+    text.into()
+}
+
+impl<'a> TreeSink for TtTreeSink<'a> {
+    fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
+        if kind == L_DOLLAR || kind == R_DOLLAR {
+            self.cursor = self.cursor.bump_subtree();
+            return;
+        }
+        if kind == LIFETIME {
+            n_tokens = 2;
+        }
+
+        let mut last = self.cursor;
+        for _ in 0..n_tokens {
+            if self.cursor.eof() {
+                break;
+            }
+            last = self.cursor;
+            let text: SmolStr = match self.cursor.token_tree() {
+                Some(tt::TokenTree::Leaf(leaf)) => {
+                    // Mark the range if needed
+                    let (text, id) = match leaf {
+                        tt::Leaf::Ident(ident) => (ident.text.clone(), ident.id),
+                        tt::Leaf::Punct(punct) => {
+                            (SmolStr::new_inline_from_ascii(1, &[punct.char as u8]), punct.id)
+                        }
+                        tt::Leaf::Literal(lit) => (lit.text.clone(), lit.id),
+                    };
+                    let range = TextRange::at(self.text_pos, TextSize::of(text.as_str()));
+                    self.token_map.insert(id, range);
+                    self.cursor = self.cursor.bump();
+                    text
+                }
+                Some(tt::TokenTree::Subtree(subtree)) => {
+                    self.cursor = self.cursor.subtree().unwrap();
+                    if let Some(id) = subtree.delimiter.map(|it| it.id) {
+                        self.open_delims.insert(id, self.text_pos);
+                    }
+                    delim_to_str(subtree.delimiter_kind(), false)
+                }
+                None => {
+                    if let Some(parent) = self.cursor.end() {
+                        self.cursor = self.cursor.bump();
+                        if let Some(id) = parent.delimiter.map(|it| it.id) {
+                            if let Some(open_delim) = self.open_delims.get(&id) {
+                                let open_range = TextRange::at(*open_delim, TextSize::of('('));
+                                let close_range = TextRange::at(self.text_pos, TextSize::of('('));
+                                self.token_map.insert_delim(id, open_range, close_range);
+                            }
+                        }
+                        delim_to_str(parent.delimiter_kind(), true)
+                    } else {
+                        continue;
+                    }
+                }
+            };
+            self.buf += &text;
+            self.text_pos += TextSize::of(text.as_str());
+        }
+
+        let text = SmolStr::new(self.buf.as_str());
+        self.buf.clear();
+        self.inner.token(kind, text);
+
+        // Add whitespace between adjoint puncts
+        let next = last.bump();
+        if let (
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(curr))),
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(_))),
+        ) = (last.token_tree(), next.token_tree())
+        {
+            // Note: We always assume the semi-colon would be the last token in
+            // other parts of RA such that we don't add whitespace here.
+            if curr.spacing == tt::Spacing::Alone && curr.char != ';' {
+                self.inner.token(WHITESPACE, " ".into());
+                self.text_pos += TextSize::of(' ');
+            }
+        }
+    }
+
+    fn start_node(&mut self, kind: SyntaxKind) {
+        self.inner.start_node(kind);
+
+        match self.roots.last_mut() {
+            None | Some(0) => self.roots.push(1),
+            Some(ref mut n) => **n += 1,
+        };
+    }
+
+    fn finish_node(&mut self) {
+        self.inner.finish_node();
+        *self.roots.last_mut().unwrap() -= 1;
+    }
+
+    fn error(&mut self, error: ParseError) {
+        self.inner.error(error, self.text_pos)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::tests::parse_macro;
+    use parser::TokenSource;
+    use syntax::{
+        algo::{insert_children, InsertPosition},
+        ast::AstNode,
+    };
+
+    #[test]
+    fn convert_tt_token_source() {
+        let expansion = parse_macro(
+            r#"
+            macro_rules! literals {
+                ($i:ident) => {
+                    {
+                        let a = 'c';
+                        let c = 1000;
+                        let f = 12E+99_f64;
+                        let s = "rust1";
+                    }
+                }
+            }
+            "#,
+        )
+        .expand_tt("literals!(foo);");
+        let tts = &[expansion.into()];
+        let buffer = tt::buffer::TokenBuffer::new(tts);
+        let mut tt_src = SubtreeTokenSource::new(&buffer);
+        let mut tokens = vec![];
+        while tt_src.current().kind != EOF {
+            tokens.push((tt_src.current().kind, tt_src.text()));
+            tt_src.bump();
+        }
+
+        // [${]
+        // [let] [a] [=] ['c'] [;]
+        assert_eq!(tokens[2 + 3].1, "'c'");
+        assert_eq!(tokens[2 + 3].0, CHAR);
+        // [let] [c] [=] [1000] [;]
+        assert_eq!(tokens[2 + 5 + 3].1, "1000");
+        assert_eq!(tokens[2 + 5 + 3].0, INT_NUMBER);
+        // [let] [f] [=] [12E+99_f64] [;]
+        assert_eq!(tokens[2 + 10 + 3].1, "12E+99_f64");
+        assert_eq!(tokens[2 + 10 + 3].0, FLOAT_NUMBER);
+
+        // [let] [s] [=] ["rust1"] [;]
+        assert_eq!(tokens[2 + 15 + 3].1, "\"rust1\"");
+        assert_eq!(tokens[2 + 15 + 3].0, STRING);
+    }
+
+    #[test]
+    fn stmts_token_trees_to_expr_is_err() {
+        let expansion = parse_macro(
+            r#"
+            macro_rules! stmts {
+                () => {
+                    let a = 0;
+                    let b = 0;
+                    let c = 0;
+                    let d = 0;
+                }
+            }
+            "#,
+        )
+        .expand_tt("stmts!();");
+        assert!(token_tree_to_syntax_node(&expansion, FragmentKind::Expr).is_err());
+    }
+
+    #[test]
+    fn test_token_tree_last_child_is_white_space() {
+        let source_file = ast::SourceFile::parse("f!({} );").ok().unwrap();
+        let macro_call = source_file.syntax().descendants().find_map(ast::MacroCall::cast).unwrap();
+        let token_tree = macro_call.token_tree().unwrap();
+
+        // Token Tree now is :
+        // TokenTree
+        // - T!['(']
+        // - TokenTree
+        //   - T!['{']
+        //   - T!['}']
+        // - WHITE_SPACE
+        // - T![')']
+
+        let rbrace =
+            token_tree.syntax().descendants_with_tokens().find(|it| it.kind() == T!['}']).unwrap();
+        let space = token_tree
+            .syntax()
+            .descendants_with_tokens()
+            .find(|it| it.kind() == SyntaxKind::WHITESPACE)
+            .unwrap();
+
+        // reorder th white space, such that the white is inside the inner token-tree.
+        let token_tree = insert_children(
+            &rbrace.parent().unwrap(),
+            InsertPosition::Last,
+            std::iter::once(space),
+        );
+
+        // Token Tree now is :
+        // TokenTree
+        // - T!['{']
+        // - T!['}']
+        // - WHITE_SPACE
+        let token_tree = ast::TokenTree::cast(token_tree).unwrap();
+        let tt = ast_to_token_tree(&token_tree).unwrap().0;
+
+        assert_eq!(tt.delimiter_kind(), Some(tt::DelimiterKind::Brace));
+    }
+
+    #[test]
+    fn test_token_tree_multi_char_punct() {
+        let source_file = ast::SourceFile::parse("struct Foo { a: x::Y }").ok().unwrap();
+        let struct_def = source_file.syntax().descendants().find_map(ast::Struct::cast).unwrap();
+        let tt = ast_to_token_tree(&struct_def).unwrap().0;
+        token_tree_to_syntax_node(&tt, FragmentKind::Item).unwrap();
+    }
+}

crates/mbe/src/tt_iter.rs

@@ -0,0 +1,75 @@
+//! FIXME: write short doc here
+
+#[derive(Debug, Clone)]
+pub(crate) struct TtIter<'a> {
+    pub(crate) inner: std::slice::Iter<'a, tt::TokenTree>,
+}
+
+impl<'a> TtIter<'a> {
+    pub(crate) fn new(subtree: &'a tt::Subtree) -> TtIter<'a> {
+        TtIter { inner: subtree.token_trees.iter() }
+    }
+
+    pub(crate) fn expect_char(&mut self, char: char) -> Result<(), ()> {
+        match self.next() {
+            Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: c, .. }))) if *c == char => {
+                Ok(())
+            }
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_subtree(&mut self) -> Result<&'a tt::Subtree, ()> {
+        match self.next() {
+            Some(tt::TokenTree::Subtree(it)) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_leaf(&mut self) -> Result<&'a tt::Leaf, ()> {
+        match self.next() {
+            Some(tt::TokenTree::Leaf(it)) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Ident(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_literal(&mut self) -> Result<&'a tt::Leaf, ()> {
+        let it = self.expect_leaf()?;
+        match it {
+            tt::Leaf::Literal(_) => Ok(it),
+            tt::Leaf::Ident(ident) if ident.text == "true" || ident.text == "false" => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_punct(&mut self) -> Result<&'a tt::Punct, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Punct(it) => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn peek_n(&self, n: usize) -> Option<&tt::TokenTree> {
+        self.inner.as_slice().get(n)
+    }
+}
+
+impl<'a> Iterator for TtIter<'a> {
+    type Item = &'a tt::TokenTree;
+    fn next(&mut self) -> Option<Self::Item> {
+        self.inner.next()
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+}
+
+impl<'a> std::iter::ExactSizeIterator for TtIter<'a> {}