preserve token spacing

Cargo.lock

@@ -1022,7 +1022,6 @@ dependencies = [
 name = "ra_macros"
 version = "0.1.0"
 dependencies = [
- "join_to_string 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "smol_str 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

crates/ra_hir/src/macros.rs

@@ -218,14 +218,28 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
             continue;
         }
         if child.kind().is_punct() {
-            let leaves = child
+            let mut prev = None;
-                .leaf_text()
+            for char in child.leaf_text().unwrap().chars() {
-                .unwrap()
+                if let Some(char) = prev {
-                .chars()
+                    token_trees.push(
-                .map(|char| tt::Punct { char })
+                        tt::Leaf::from(tt::Punct {
-                .map(tt::Leaf::from)
+                            char,
-                .map(tt::TokenTree::from);
+                            spacing: tt::Spacing::Joint,
-            token_trees.extend(leaves);
+                        })
+                        .into(),
+                    );
+                }
+                prev = Some(char)
+            }
+            if let Some(char) = prev {
+                token_trees.push(
+                    tt::Leaf::from(tt::Punct {
+                        char,
+                        spacing: tt::Spacing::Alone,
+                    })
+                    .into(),
+                );
+            }
         } else {
             let child: tt::TokenTree = if child.kind() == TOKEN_TREE {
                 convert_tt(child)?.into()
@@ -254,7 +268,7 @@ fn convert_tt(tt: &SyntaxNode) -> Option<tt::Subtree> {
 
 #[test]
 fn test_convert_tt() {
-    let macro_defenition = r#"
+    let macro_definition = r#"
 macro_rules! impl_froms {
     ($e:ident: $($v:ident),*) => {
         $(
@@ -272,8 +286,8 @@ macro_rules! impl_froms {
 impl_froms!(TokenTree: Leaf, Subtree);
 "#;
 
-    let source_file = ast::SourceFile::parse(macro_defenition);
+    let source_file = ast::SourceFile::parse(macro_definition);
-    let macro_defenition = source_file
+    let macro_definition = source_file
         .syntax()
         .descendants()
         .find_map(ast::MacroCall::cast)
@@ -286,9 +300,9 @@ impl_froms!(TokenTree: Leaf, Subtree);
         .find_map(ast::MacroCall::cast)
         .unwrap();
 
-    let defenition_tt = macro_call_to_tt(macro_defenition).unwrap();
+    let definition_tt = macro_call_to_tt(macro_definition).unwrap();
     let invocation_tt = macro_call_to_tt(macro_invocation).unwrap();
-    let mbe = mbe::parse(&defenition_tt).unwrap();
+    let mbe = mbe::parse(&definition_tt).unwrap();
     let expansion = mbe::exapnd(&mbe, &invocation_tt).unwrap();
     assert_eq!(
         expansion.to_string(),

crates/ra_macros/Cargo.toml

@@ -7,4 +7,3 @@ authors = ["Aleksey Kladov <aleksey.kladov@gmail.com>"]
 [dependencies]
 rustc-hash = "1.0.0"
 smol_str = "0.1.9"
-join_to_string = "0.1.3"

crates/ra_macros/src/mbe.rs

@@ -45,7 +45,7 @@ pub(crate) struct Subtree {
 pub(crate) struct Repeat {
     pub(crate) subtree: Subtree,
     pub(crate) kind: RepeatKind,
-    pub(crate) separator: Option<Punct>,
+    pub(crate) separator: Option<char>,
 }
 
 #[derive(Debug)]

crates/ra_macros/src/mbe_parser.rs

@@ -28,16 +28,14 @@ fn parse_subtree(tt: &tt::Subtree) -> Option<mbe::Subtree> {
     while let Some(tt) = p.eat() {
         let child: mbe::TokenTree = match tt {
             tt::TokenTree::Leaf(leaf) => match leaf {
-                tt::Leaf::Punct(tt::Punct { char: '$' }) => {
+                tt::Leaf::Punct(tt::Punct { char: '$', .. }) => {
                     if p.at_ident().is_some() {
                         mbe::Leaf::from(parse_var(&mut p)?).into()
                     } else {
                         parse_repeat(&mut p)?.into()
                     }
                 }
-                tt::Leaf::Punct(tt::Punct { char }) => {
+                tt::Leaf::Punct(punct) => mbe::Leaf::from(*punct).into(),
-                    mbe::Leaf::from(mbe::Punct { char: *char }).into()
-                }
                 tt::Leaf::Ident(tt::Ident { text }) => {
                     mbe::Leaf::from(mbe::Ident { text: text.clone() }).into()
                 }
@@ -78,7 +76,7 @@ fn parse_repeat(p: &mut TtCursor) -> Option<mbe::Repeat> {
     let sep = p.eat_punct()?;
     let (separator, rep) = match sep.char {
         '*' | '+' | '?' => (None, sep.char),
-        char => (Some(mbe::Punct { char }), p.eat_punct()?.char),
+        char => (Some(char), p.eat_punct()?.char),
     };
 
     let kind = match rep {

crates/ra_macros/src/tt.rs

@@ -1,7 +1,6 @@
 use std::fmt;
 
 use smol_str::SmolStr;
-use join_to_string::join;
 
 #[derive(Debug, Clone)]
 pub enum TokenTree {
@@ -37,9 +36,16 @@ pub struct Literal {
     pub text: SmolStr,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Punct {
     pub char: char,
+    pub spacing: Spacing,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Spacing {
+    Alone,
+    Joint,
 }
 
 #[derive(Debug, Clone)]
@@ -64,10 +70,23 @@ impl fmt::Display for Subtree {
             Delimiter::Bracket => ("[", "]"),
             Delimiter::None => ("", ""),
         };
-        join(self.token_trees.iter())
+        f.write_str(l)?;
-            .separator(" ")
+        let mut needs_space = false;
-            .surround_with(l, r)
+        for tt in self.token_trees.iter() {
-            .to_fmt(f)
+            if needs_space {
+                f.write_str(" ")?;
+            }
+            needs_space = true;
+            match tt {
+                TokenTree::Leaf(Leaf::Punct(p)) => {
+                    needs_space = p.spacing == Spacing::Alone;
+                    fmt::Display::fmt(p, f)?
+                }
+                tt => fmt::Display::fmt(tt, f)?,
+            }
+        }
+        f.write_str(r)?;
+        Ok(())
     }
 }
 

crates/ra_macros/src/tt_cursor.rs

@@ -28,7 +28,7 @@ impl<'a> TtCursor<'a> {
 
     pub(crate) fn at_char(&self, char: char) -> bool {
         match self.at_punct() {
-            Some(tt::Punct { char: c }) if *c == char => true,
+            Some(tt::Punct { char: c, .. }) if *c == char => true,
             _ => false,
         }
     }