Switch token trees to use Symbols

crates/proc-macro-srv/src/lib.rs

@@ -130,14 +130,13 @@ impl ProcMacroSrvSpan for TokenId {
     type Server = server_impl::token_id::TokenIdServer;
 
     fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server {
-        Self::Server { interner: &server_impl::SYMBOL_INTERNER, call_site, def_site, mixed_site }
+        Self::Server { call_site, def_site, mixed_site }
     }
 }
 impl ProcMacroSrvSpan for Span {
     type Server = server_impl::rust_analyzer_span::RaSpanServer;
     fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server {
         Self::Server {
-            interner: &server_impl::SYMBOL_INTERNER,
             call_site,
             def_site,
             mixed_site,

crates/proc-macro-srv/src/server_impl.rs

@@ -14,9 +14,9 @@ mod token_stream;
 pub use token_stream::TokenStream;
 
 pub mod rust_analyzer_span;
-mod symbol;
+// mod symbol;
 pub mod token_id;
-pub use symbol::*;
+// pub use symbol::*;
 use tt::Spacing;
 
 fn delim_to_internal<S>(d: proc_macro::Delimiter, span: bridge::DelimSpan<S>) -> tt::Delimiter<S> {

crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs

@@ -10,13 +10,14 @@ use std::{
     ops::{Bound, Range},
 };
 
+use intern::Symbol;
 use proc_macro::bridge::{self, server};
 use span::{Span, FIXUP_ERASED_FILE_AST_ID_MARKER};
 use tt::{TextRange, TextSize};
 
 use crate::server_impl::{
     delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal,
-    token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    token_stream::TokenStreamBuilder,
 };
 mod tt {
     pub use tt::*;
@@ -36,7 +37,6 @@ pub struct SourceFile;
 pub struct FreeFunctions;
 
 pub struct RaSpanServer {
-    pub(crate) interner: SymbolInternerRef,
     // FIXME: Report this back to the caller to track as dependencies
     pub tracked_env_vars: HashMap<Box<str>, Option<Box<str>>>,
     // FIXME: Report this back to the caller to track as dependencies
@@ -126,15 +126,10 @@ impl server::FreeFunctions for RaSpanServer {
         let lit = &lit[start_offset..lit.len() - end_offset];
         let suffix = match suffix {
             "" | "_" => None,
-            suffix => Some(Symbol::intern(self.interner, suffix)),
+            suffix => Some(Symbol::intern(suffix)),
         };
 
-        Ok(bridge::Literal {
-            kind,
-            symbol: Symbol::intern(self.interner, lit),
-            suffix,
-            span: self.call_site,
-        })
+        Ok(bridge::Literal { kind, symbol: Symbol::intern(lit), suffix, span: self.call_site })
     }
 
     fn emit_diagnostic(&mut self, _: bridge::Diagnostic<Self::Span>) {
@@ -170,9 +165,9 @@ impl server::TokenStream for RaSpanServer {
             }
 
             bridge::TokenTree::Ident(ident) => {
-                let text = ident.sym.text(self.interner);
+                let text = ident.sym;
                 let ident: tt::Ident = tt::Ident {
-                    text,
+                    sym: text,
                     span: ident.span,
                     is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No },
                 };
@@ -183,8 +178,8 @@ impl server::TokenStream for RaSpanServer {
 
             bridge::TokenTree::Literal(literal) => {
                 let literal = tt::Literal {
-                    text: literal.symbol.text(self.interner),
-                    suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))),
+                    symbol: literal.symbol,
+                    suffix: literal.suffix,
                     span: literal.span,
                     kind: literal_kind_to_internal(literal.kind),
                 };
@@ -255,7 +250,7 @@ impl server::TokenStream for RaSpanServer {
             .map(|tree| match tree {
                 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
                     bridge::TokenTree::Ident(bridge::Ident {
-                        sym: Symbol::intern(self.interner, &ident.text),
+                        sym: ident.sym,
                         is_raw: ident.is_raw.yes(),
                         span: ident.span,
                     })
@@ -264,8 +259,8 @@ impl server::TokenStream for RaSpanServer {
                     bridge::TokenTree::Literal(bridge::Literal {
                         span: lit.span,
                         kind: literal_kind_to_external(lit.kind),
-                        symbol: Symbol::intern(self.interner, &lit.text),
-                        suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)),
+                        symbol: lit.symbol,
+                        suffix: lit.suffix,
                     })
                 }
                 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
@@ -464,12 +459,95 @@ impl server::Server for RaSpanServer {
     }
 
     fn intern_symbol(ident: &str) -> Self::Symbol {
-        // FIXME: should be `self.interner` once the proc-macro api allows it.
-        Symbol::intern(&SYMBOL_INTERNER, &::tt::SmolStr::from(ident))
+        Symbol::intern(ident)
     }
 
     fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
-        // FIXME: should be `self.interner` once the proc-macro api allows it.
-        f(symbol.text(&SYMBOL_INTERNER).as_str())
+        f(symbol.as_str())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use span::SyntaxContextId;
+
+    use super::*;
+
+    #[test]
+    fn test_ra_server_to_string() {
+        let span = Span {
+            range: TextRange::empty(TextSize::new(0)),
+            anchor: span::SpanAnchor {
+                file_id: span::FileId::from_raw(0),
+                ast_id: span::ErasedFileAstId::from_raw(0.into()),
+            },
+            ctx: SyntaxContextId::ROOT,
+        };
+        let s = TokenStream {
+            token_trees: vec![
+                tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                    sym: Symbol::intern("struct"),
+                    span,
+                    is_raw: tt::IdentIsRaw::No,
+                })),
+                tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                    sym: Symbol::intern("T"),
+                    span: span,
+                    is_raw: tt::IdentIsRaw::No,
+                })),
+                tt::TokenTree::Subtree(tt::Subtree {
+                    delimiter: tt::Delimiter {
+                        open: span,
+                        close: span,
+                        kind: tt::DelimiterKind::Brace,
+                    },
+                    token_trees: Box::new([]),
+                }),
+            ],
+        };
+
+        assert_eq!(s.to_string(), "struct T {}");
+    }
+
+    #[test]
+    fn test_ra_server_from_str() {
+        let span = Span {
+            range: TextRange::empty(TextSize::new(0)),
+            anchor: span::SpanAnchor {
+                file_id: span::FileId::from_raw(0),
+                ast_id: span::ErasedFileAstId::from_raw(0.into()),
+            },
+            ctx: SyntaxContextId::ROOT,
+        };
+        let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
+            delimiter: tt::Delimiter {
+                open: span,
+                close: span,
+                kind: tt::DelimiterKind::Parenthesis,
+            },
+            token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                is_raw: tt::IdentIsRaw::No,
+                sym: Symbol::intern("a"),
+                span,
+            }))]),
+        });
+
+        let t1 = TokenStream::from_str("(a)", span).unwrap();
+        assert_eq!(t1.token_trees.len(), 1);
+        assert_eq!(t1.token_trees[0], subtree_paren_a);
+
+        let t2 = TokenStream::from_str("(a);", span).unwrap();
+        assert_eq!(t2.token_trees.len(), 2);
+        assert_eq!(t2.token_trees[0], subtree_paren_a);
+
+        let underscore = TokenStream::from_str("_", span).unwrap();
+        assert_eq!(
+            underscore.token_trees[0],
+            tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                sym: Symbol::intern("_"),
+                span,
+                is_raw: tt::IdentIsRaw::No,
+            }))
+        );
     }
 }

crates/proc-macro-srv/src/server_impl/symbol.rs

@@ -1,7 +1,6 @@
 //! Symbol interner for proc-macro-srv
 
 use std::{cell::RefCell, collections::HashMap, thread::LocalKey};
-use tt::SmolStr;
 
 thread_local! {
     pub(crate) static SYMBOL_INTERNER: RefCell<SymbolInterner> = Default::default();

crates/proc-macro-srv/src/server_impl/token_id.rs

@@ -5,11 +5,12 @@ use std::{
     ops::{Bound, Range},
 };
 
+use intern::Symbol;
 use proc_macro::bridge::{self, server};
 
 use crate::server_impl::{
     delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal,
-    token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    token_stream::TokenStreamBuilder,
 };
 mod tt {
     pub use proc_macro_api::msg::TokenId;
@@ -36,7 +37,6 @@ pub struct SourceFile;
 pub struct FreeFunctions;
 
 pub struct TokenIdServer {
-    pub(crate) interner: SymbolInternerRef,
     pub call_site: Span,
     pub def_site: Span,
     pub mixed_site: Span,
@@ -117,15 +117,10 @@ impl server::FreeFunctions for TokenIdServer {
         let lit = &lit[start_offset..lit.len() - end_offset];
         let suffix = match suffix {
             "" | "_" => None,
-            suffix => Some(Symbol::intern(self.interner, suffix)),
+            suffix => Some(Symbol::intern(suffix)),
         };
 
-        Ok(bridge::Literal {
-            kind,
-            symbol: Symbol::intern(self.interner, lit),
-            suffix,
-            span: self.call_site,
-        })
+        Ok(bridge::Literal { kind, symbol: Symbol::intern(lit), suffix, span: self.call_site })
     }
 
     fn emit_diagnostic(&mut self, _: bridge::Diagnostic<Self::Span>) {}
@@ -159,9 +154,8 @@ impl server::TokenStream for TokenIdServer {
             }
 
             bridge::TokenTree::Ident(ident) => {
-                let text = ident.sym.text(self.interner);
                 let ident: tt::Ident = tt::Ident {
-                    text,
+                    sym: ident.sym,
                     span: ident.span,
                     is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No },
                 };
@@ -172,8 +166,8 @@ impl server::TokenStream for TokenIdServer {
 
             bridge::TokenTree::Literal(literal) => {
                 let literal = Literal {
-                    text: literal.symbol.text(self.interner),
-                    suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))),
+                    symbol: literal.symbol,
+                    suffix: literal.suffix,
                     span: literal.span,
                     kind: literal_kind_to_internal(literal.kind),
                 };
@@ -239,7 +233,7 @@ impl server::TokenStream for TokenIdServer {
             .map(|tree| match tree {
                 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
                     bridge::TokenTree::Ident(bridge::Ident {
-                        sym: Symbol::intern(self.interner, &ident.text),
+                        sym: ident.sym,
                         is_raw: ident.is_raw.yes(),
                         span: ident.span,
                     })
@@ -248,8 +242,8 @@ impl server::TokenStream for TokenIdServer {
                     bridge::TokenTree::Literal(bridge::Literal {
                         span: lit.span,
                         kind: literal_kind_to_external(lit.kind),
-                        symbol: Symbol::intern(self.interner, &lit.text),
-                        suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)),
+                        symbol: lit.symbol,
+                        suffix: lit.suffix,
                     })
                 }
                 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
@@ -366,11 +360,11 @@ impl server::Server for TokenIdServer {
     }
 
     fn intern_symbol(ident: &str) -> Self::Symbol {
-        Symbol::intern(&SYMBOL_INTERNER, &::tt::SmolStr::from(ident))
+        Symbol::intern(ident)
     }
 
     fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
-        f(symbol.text(&SYMBOL_INTERNER).as_str())
+        f(symbol.as_str())
     }
 }
 
@@ -383,12 +377,12 @@ mod tests {
         let s = TokenStream {
             token_trees: vec![
                 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
-                    text: "struct".into(),
+                    sym: Symbol::intern("struct"),
                     span: tt::TokenId(0),
                     is_raw: tt::IdentIsRaw::No,
                 })),
                 tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
-                    text: "T".into(),
+                    sym: Symbol::intern("T"),
                     span: tt::TokenId(0),
                     is_raw: tt::IdentIsRaw::No,
                 })),
@@ -416,7 +410,7 @@ mod tests {
             },
             token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
                 is_raw: tt::IdentIsRaw::No,
-                text: "a".into(),
+                sym: Symbol::intern("a"),
                 span: tt::TokenId(0),
             }))]),
         });
@@ -433,7 +427,7 @@ mod tests {
         assert_eq!(
             underscore.token_trees[0],
             tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
-                text: "_".into(),
+                sym: Symbol::intern("_"),
                 span: tt::TokenId(0),
                 is_raw: tt::IdentIsRaw::No,
             }))