Switch token trees to use Symbols

crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs

@@ -10,13 +10,14 @@ use std::{
     ops::{Bound, Range},
 };
 
+use intern::Symbol;
 use proc_macro::bridge::{self, server};
 use span::{Span, FIXUP_ERASED_FILE_AST_ID_MARKER};
 use tt::{TextRange, TextSize};
 
 use crate::server_impl::{
     delim_to_external, delim_to_internal, literal_kind_to_external, literal_kind_to_internal,
-    token_stream::TokenStreamBuilder, Symbol, SymbolInternerRef, SYMBOL_INTERNER,
+    token_stream::TokenStreamBuilder,
 };
 mod tt {
     pub use tt::*;
@@ -36,7 +37,6 @@ pub struct SourceFile;
 pub struct FreeFunctions;
 
 pub struct RaSpanServer {
-    pub(crate) interner: SymbolInternerRef,
     // FIXME: Report this back to the caller to track as dependencies
     pub tracked_env_vars: HashMap<Box<str>, Option<Box<str>>>,
     // FIXME: Report this back to the caller to track as dependencies
@@ -126,15 +126,10 @@ impl server::FreeFunctions for RaSpanServer {
         let lit = &lit[start_offset..lit.len() - end_offset];
         let suffix = match suffix {
             "" | "_" => None,
-            suffix => Some(Symbol::intern(self.interner, suffix)),
+            suffix => Some(Symbol::intern(suffix)),
         };
 
-        Ok(bridge::Literal {
-            kind,
-            symbol: Symbol::intern(self.interner, lit),
-            suffix,
-            span: self.call_site,
-        })
+        Ok(bridge::Literal { kind, symbol: Symbol::intern(lit), suffix, span: self.call_site })
     }
 
     fn emit_diagnostic(&mut self, _: bridge::Diagnostic<Self::Span>) {
@@ -170,9 +165,9 @@ impl server::TokenStream for RaSpanServer {
             }
 
             bridge::TokenTree::Ident(ident) => {
-                let text = ident.sym.text(self.interner);
+                let text = ident.sym;
                 let ident: tt::Ident = tt::Ident {
-                    text,
+                    sym: text,
                     span: ident.span,
                     is_raw: if ident.is_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No },
                 };
@@ -183,8 +178,8 @@ impl server::TokenStream for RaSpanServer {
 
             bridge::TokenTree::Literal(literal) => {
                 let literal = tt::Literal {
-                    text: literal.symbol.text(self.interner),
-                    suffix: literal.suffix.map(|it| Box::new(it.text(self.interner))),
+                    symbol: literal.symbol,
+                    suffix: literal.suffix,
                     span: literal.span,
                     kind: literal_kind_to_internal(literal.kind),
                 };
@@ -255,7 +250,7 @@ impl server::TokenStream for RaSpanServer {
             .map(|tree| match tree {
                 tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
                     bridge::TokenTree::Ident(bridge::Ident {
-                        sym: Symbol::intern(self.interner, &ident.text),
+                        sym: ident.sym,
                         is_raw: ident.is_raw.yes(),
                         span: ident.span,
                     })
@@ -264,8 +259,8 @@ impl server::TokenStream for RaSpanServer {
                     bridge::TokenTree::Literal(bridge::Literal {
                         span: lit.span,
                         kind: literal_kind_to_external(lit.kind),
-                        symbol: Symbol::intern(self.interner, &lit.text),
-                        suffix: lit.suffix.map(|it| Symbol::intern(self.interner, &it)),
+                        symbol: lit.symbol,
+                        suffix: lit.suffix,
                     })
                 }
                 tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => {
@@ -464,12 +459,95 @@ impl server::Server for RaSpanServer {
     }
 
     fn intern_symbol(ident: &str) -> Self::Symbol {
-        // FIXME: should be `self.interner` once the proc-macro api allows it.
-        Symbol::intern(&SYMBOL_INTERNER, &::tt::SmolStr::from(ident))
+        Symbol::intern(ident)
     }
 
     fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
-        // FIXME: should be `self.interner` once the proc-macro api allows it.
-        f(symbol.text(&SYMBOL_INTERNER).as_str())
+        f(symbol.as_str())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use span::SyntaxContextId;
+
+    use super::*;
+
+    #[test]
+    fn test_ra_server_to_string() {
+        let span = Span {
+            range: TextRange::empty(TextSize::new(0)),
+            anchor: span::SpanAnchor {
+                file_id: span::FileId::from_raw(0),
+                ast_id: span::ErasedFileAstId::from_raw(0.into()),
+            },
+            ctx: SyntaxContextId::ROOT,
+        };
+        let s = TokenStream {
+            token_trees: vec![
+                tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                    sym: Symbol::intern("struct"),
+                    span,
+                    is_raw: tt::IdentIsRaw::No,
+                })),
+                tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                    sym: Symbol::intern("T"),
+                    span: span,
+                    is_raw: tt::IdentIsRaw::No,
+                })),
+                tt::TokenTree::Subtree(tt::Subtree {
+                    delimiter: tt::Delimiter {
+                        open: span,
+                        close: span,
+                        kind: tt::DelimiterKind::Brace,
+                    },
+                    token_trees: Box::new([]),
+                }),
+            ],
+        };
+
+        assert_eq!(s.to_string(), "struct T {}");
+    }
+
+    #[test]
+    fn test_ra_server_from_str() {
+        let span = Span {
+            range: TextRange::empty(TextSize::new(0)),
+            anchor: span::SpanAnchor {
+                file_id: span::FileId::from_raw(0),
+                ast_id: span::ErasedFileAstId::from_raw(0.into()),
+            },
+            ctx: SyntaxContextId::ROOT,
+        };
+        let subtree_paren_a = tt::TokenTree::Subtree(tt::Subtree {
+            delimiter: tt::Delimiter {
+                open: span,
+                close: span,
+                kind: tt::DelimiterKind::Parenthesis,
+            },
+            token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                is_raw: tt::IdentIsRaw::No,
+                sym: Symbol::intern("a"),
+                span,
+            }))]),
+        });
+
+        let t1 = TokenStream::from_str("(a)", span).unwrap();
+        assert_eq!(t1.token_trees.len(), 1);
+        assert_eq!(t1.token_trees[0], subtree_paren_a);
+
+        let t2 = TokenStream::from_str("(a);", span).unwrap();
+        assert_eq!(t2.token_trees.len(), 2);
+        assert_eq!(t2.token_trees[0], subtree_paren_a);
+
+        let underscore = TokenStream::from_str("_", span).unwrap();
+        assert_eq!(
+            underscore.token_trees[0],
+            tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                sym: Symbol::intern("_"),
+                span,
+                is_raw: tt::IdentIsRaw::No,
+            }))
+        );
     }
 }