Update 1.63 proc macro ABI to match rustc

crates/proc-macro-srv/src/abis/abi_1_63/proc_macro/bridge/client.rs

@@ -178,8 +178,6 @@ define_handles! {
     'owned:
     FreeFunctions,
     TokenStream,
-    TokenStreamBuilder,
-    TokenStreamIter,
     Group,
     Literal,
     SourceFile,
@@ -204,12 +202,6 @@ impl Clone for TokenStream {
     }
 }
 
-impl Clone for TokenStreamIter {
-    fn clone(&self) -> Self {
-        self.clone()
-    }
-}
-
 impl Clone for Group {
     fn clone(&self) -> Self {
         self.clone()

crates/proc-macro-srv/src/abis/abi_1_63/proc_macro/bridge/handle.rs

@@ -1,7 +1,7 @@
 //! Server-side handles and storage for per-handle data.
 
 use std::collections::{BTreeMap, HashMap};
-use std::hash::Hash;
+use std::hash::{BuildHasher, Hash};
 use std::num::NonZeroU32;
 use std::ops::{Index, IndexMut};
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -51,15 +51,31 @@ impl<T> IndexMut<Handle> for OwnedStore<T> {
     }
 }
 
+// HACK(eddyb) deterministic `std::collections::hash_map::RandomState` replacement
+// that doesn't require adding any dependencies to `proc_macro` (like `rustc-hash`).
+#[derive(Clone)]
+struct NonRandomState;
+
+impl BuildHasher for NonRandomState {
+    type Hasher = std::collections::hash_map::DefaultHasher;
+    #[inline]
+    fn build_hasher(&self) -> Self::Hasher {
+        Self::Hasher::new()
+    }
+}
+
 /// Like `OwnedStore`, but avoids storing any value more than once.
 pub(super) struct InternedStore<T: 'static> {
     owned: OwnedStore<T>,
-    interner: HashMap<T, Handle>,
+    interner: HashMap<T, Handle, NonRandomState>,
 }
 
 impl<T: Copy + Eq + Hash> InternedStore<T> {
     pub(super) fn new(counter: &'static AtomicUsize) -> Self {
-        InternedStore { owned: OwnedStore::new(counter), interner: HashMap::new() }
+        InternedStore {
+            owned: OwnedStore::new(counter),
+            interner: HashMap::with_hasher(NonRandomState),
+        }
     }
 
     pub(super) fn alloc(&mut self, x: T) -> Handle {

crates/proc-macro-srv/src/abis/abi_1_63/proc_macro/bridge/mod.rs

@@ -60,7 +60,6 @@ macro_rules! with_api {
             TokenStream {
                 fn drop($self: $S::TokenStream);
                 fn clone($self: &$S::TokenStream) -> $S::TokenStream;
-                fn new() -> $S::TokenStream;
                 fn is_empty($self: &$S::TokenStream) -> bool;
                 fn expand_expr($self: &$S::TokenStream) -> Result<$S::TokenStream, ()>;
                 fn from_str(src: &str) -> $S::TokenStream;
@@ -68,25 +67,22 @@ macro_rules! with_api {
                 fn from_token_tree(
                     tree: TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>,
                 ) -> $S::TokenStream;
-                fn into_iter($self: $S::TokenStream) -> $S::TokenStreamIter;
+                fn concat_trees(
-            },
+                    base: Option<$S::TokenStream>,
-            TokenStreamBuilder {
+                    trees: Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>,
-                fn drop($self: $S::TokenStreamBuilder);
+                ) -> $S::TokenStream;
-                fn new() -> $S::TokenStreamBuilder;
+                fn concat_streams(
-                fn push($self: &mut $S::TokenStreamBuilder, stream: $S::TokenStream);
+                    base: Option<$S::TokenStream>,
-                fn build($self: $S::TokenStreamBuilder) -> $S::TokenStream;
+                    streams: Vec<$S::TokenStream>,
-            },
+                ) -> $S::TokenStream;
-            TokenStreamIter {
+                fn into_trees(
-                fn drop($self: $S::TokenStreamIter);
+                    $self: $S::TokenStream
-                fn clone($self: &$S::TokenStreamIter) -> $S::TokenStreamIter;
+                ) -> Vec<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
-                fn next(
-                    $self: &mut $S::TokenStreamIter,
-                ) -> Option<TokenTree<$S::Group, $S::Punct, $S::Ident, $S::Literal>>;
             },
             Group {
                 fn drop($self: $S::Group);
                 fn clone($self: &$S::Group) -> $S::Group;
-                fn new(delimiter: Delimiter, stream: $S::TokenStream) -> $S::Group;
+                fn new(delimiter: Delimiter, stream: Option<$S::TokenStream>) -> $S::Group;
                 fn delimiter($self: &$S::Group) -> Delimiter;
                 fn stream($self: &$S::Group) -> $S::TokenStream;
                 fn span($self: &$S::Group) -> $S::Span;
@@ -311,29 +307,18 @@ impl<'a, T, M> Unmark for &'a mut Marked<T, M> {
     }
 }
 
-impl<T: Mark> Mark for Option<T> {
+impl<T: Mark> Mark for Vec<T> {
-    type Unmarked = Option<T::Unmarked>;
+    type Unmarked = Vec<T::Unmarked>;
     fn mark(unmarked: Self::Unmarked) -> Self {
-        unmarked.map(T::mark)
+        // Should be a no-op due to std's in-place collect optimizations.
+        unmarked.into_iter().map(T::mark).collect()
     }
 }
-impl<T: Unmark> Unmark for Option<T> {
+impl<T: Unmark> Unmark for Vec<T> {
-    type Unmarked = Option<T::Unmarked>;
+    type Unmarked = Vec<T::Unmarked>;
     fn unmark(self) -> Self::Unmarked {
-        self.map(T::unmark)
+        // Should be a no-op due to std's in-place collect optimizations.
-    }
+        self.into_iter().map(T::unmark).collect()
-}
-
-impl<T: Mark, E: Mark> Mark for Result<T, E> {
-    type Unmarked = Result<T::Unmarked, E::Unmarked>;
-    fn mark(unmarked: Self::Unmarked) -> Self {
-        unmarked.map(T::mark).map_err(E::mark)
-    }
-}
-impl<T: Unmark, E: Unmark> Unmark for Result<T, E> {
-    type Unmarked = Result<T::Unmarked, E::Unmarked>;
-    fn unmark(self) -> Self::Unmarked {
-        self.map(T::unmark).map_err(E::unmark)
     }
 }
 
@@ -367,7 +352,6 @@ mark_noop! {
     Level,
     LineColumn,
     Spacing,
-    Bound<usize>,
 }
 
 rpc_encode_decode!(
@@ -394,6 +378,61 @@ rpc_encode_decode!(
     }
 );
 
+macro_rules! mark_compound {
+    (enum $name:ident <$($T:ident),+> { $($variant:ident $(($field:ident))?),* $(,)? }) => {
+        impl<$($T: Mark),+> Mark for $name <$($T),+> {
+            type Unmarked = $name <$($T::Unmarked),+>;
+            fn mark(unmarked: Self::Unmarked) -> Self {
+                match unmarked {
+                    $($name::$variant $(($field))? => {
+                        $name::$variant $((Mark::mark($field)))?
+                    })*
+                }
+            }
+        }
+
+        impl<$($T: Unmark),+> Unmark for $name <$($T),+> {
+            type Unmarked = $name <$($T::Unmarked),+>;
+            fn unmark(self) -> Self::Unmarked {
+                match self {
+                    $($name::$variant $(($field))? => {
+                        $name::$variant $((Unmark::unmark($field)))?
+                    })*
+                }
+            }
+        }
+    }
+}
+
+macro_rules! compound_traits {
+    ($($t:tt)*) => {
+        rpc_encode_decode!($($t)*);
+        mark_compound!($($t)*);
+    };
+}
+
+compound_traits!(
+    enum Bound<T> {
+        Included(x),
+        Excluded(x),
+        Unbounded,
+    }
+);
+
+compound_traits!(
+    enum Option<T> {
+        Some(t),
+        None,
+    }
+);
+
+compound_traits!(
+    enum Result<T, E> {
+        Ok(t),
+        Err(e),
+    }
+);
+
 #[derive(Clone)]
 pub enum TokenTree<G, P, I, L> {
     Group(G),
@@ -402,30 +441,7 @@ pub enum TokenTree<G, P, I, L> {
     Literal(L),
 }
 
-impl<G: Mark, P: Mark, I: Mark, L: Mark> Mark for TokenTree<G, P, I, L> {
+compound_traits!(
-    type Unmarked = TokenTree<G::Unmarked, P::Unmarked, I::Unmarked, L::Unmarked>;
-    fn mark(unmarked: Self::Unmarked) -> Self {
-        match unmarked {
-            TokenTree::Group(tt) => TokenTree::Group(G::mark(tt)),
-            TokenTree::Punct(tt) => TokenTree::Punct(P::mark(tt)),
-            TokenTree::Ident(tt) => TokenTree::Ident(I::mark(tt)),
-            TokenTree::Literal(tt) => TokenTree::Literal(L::mark(tt)),
-        }
-    }
-}
-impl<G: Unmark, P: Unmark, I: Unmark, L: Unmark> Unmark for TokenTree<G, P, I, L> {
-    type Unmarked = TokenTree<G::Unmarked, P::Unmarked, I::Unmarked, L::Unmarked>;
-    fn unmark(self) -> Self::Unmarked {
-        match self {
-            TokenTree::Group(tt) => TokenTree::Group(tt.unmark()),
-            TokenTree::Punct(tt) => TokenTree::Punct(tt.unmark()),
-            TokenTree::Ident(tt) => TokenTree::Ident(tt.unmark()),
-            TokenTree::Literal(tt) => TokenTree::Literal(tt.unmark()),
-        }
-    }
-}
-
-rpc_encode_decode!(
     enum TokenTree<G, P, I, L> {
         Group(tt),
         Punct(tt),

crates/proc-macro-srv/src/abis/abi_1_63/proc_macro/bridge/rpc.rs

@@ -4,7 +4,6 @@ use std::any::Any;
 use std::char;
 use std::io::Write;
 use std::num::NonZeroU32;
-use std::ops::Bound;
 use std::str;
 
 pub(super) type Writer = super::buffer::Buffer;
@@ -43,15 +42,17 @@ macro_rules! rpc_encode_decode {
             }
         }
     };
-    (struct $name:ident { $($field:ident),* $(,)? }) => {
+    (struct $name:ident $(<$($T:ident),+>)? { $($field:ident),* $(,)? }) => {
-        impl<S> Encode<S> for $name {
+        impl<S, $($($T: Encode<S>),+)?> Encode<S> for $name $(<$($T),+>)? {
             fn encode(self, w: &mut Writer, s: &mut S) {
                 $(self.$field.encode(w, s);)*
             }
         }
 
-        impl<S> DecodeMut<'_, '_, S> for $name {
+        impl<'a, S, $($($T: for<'s> DecodeMut<'a, 's, S>),+)?> DecodeMut<'a, '_, S>
-            fn decode(r: &mut Reader<'_>, s: &mut S) -> Self {
+            for $name $(<$($T),+>)?
+        {
+            fn decode(r: &mut Reader<'a>, s: &mut S) -> Self {
                 $name {
                     $($field: DecodeMut::decode(r, s)),*
                 }
@@ -184,28 +185,6 @@ impl<'a, S, A: for<'s> DecodeMut<'a, 's, S>, B: for<'s> DecodeMut<'a, 's, S>> De
     }
 }
 
-rpc_encode_decode!(
-    enum Bound<T> {
-        Included(x),
-        Excluded(x),
-        Unbounded,
-    }
-);
-
-rpc_encode_decode!(
-    enum Option<T> {
-        None,
-        Some(x),
-    }
-);
-
-rpc_encode_decode!(
-    enum Result<T, E> {
-        Ok(x),
-        Err(e),
-    }
-);
-
 impl<S> Encode<S> for &[u8] {
     fn encode(self, w: &mut Writer, s: &mut S) {
         self.len().encode(w, s);
@@ -246,6 +225,26 @@ impl<S> DecodeMut<'_, '_, S> for String {
     }
 }
 
+impl<S, T: Encode<S>> Encode<S> for Vec<T> {
+    fn encode(self, w: &mut Writer, s: &mut S) {
+        self.len().encode(w, s);
+        for x in self {
+            x.encode(w, s);
+        }
+    }
+}
+
+impl<'a, S, T: for<'s> DecodeMut<'a, 's, S>> DecodeMut<'a, '_, S> for Vec<T> {
+    fn decode(r: &mut Reader<'a>, s: &mut S) -> Self {
+        let len = usize::decode(r, s);
+        let mut vec = Vec::with_capacity(len);
+        for _ in 0..len {
+            vec.push(T::decode(r, s));
+        }
+        vec
+    }
+}
+
 /// Simplified version of panic payloads, ignoring
 /// types other than `&'static str` and `String`.
 pub enum PanicMessage {

crates/proc-macro-srv/src/abis/abi_1_63/proc_macro/bridge/server.rs

@@ -8,8 +8,6 @@ use super::client::HandleStore;
 pub trait Types {
     type FreeFunctions: 'static;
     type TokenStream: 'static + Clone;
-    type TokenStreamBuilder: 'static;
-    type TokenStreamIter: 'static + Clone;
     type Group: 'static + Clone;
     type Punct: 'static + Copy + Eq + Hash;
     type Ident: 'static + Copy + Eq + Hash;
@@ -275,13 +273,17 @@ fn run_server<
 }
 
 impl client::Client<super::super::TokenStream, super::super::TokenStream> {
-    pub fn run<S: Server>(
+    pub fn run<S>(
         &self,
         strategy: &impl ExecutionStrategy,
         server: S,
         input: S::TokenStream,
         force_show_panics: bool,
-    ) -> Result<S::TokenStream, PanicMessage> {
+    ) -> Result<S::TokenStream, PanicMessage>
+    where
+        S: Server,
+        S::TokenStream: Default,
+    {
         let client::Client { get_handle_counters, run, _marker } = *self;
         run_server(
             strategy,
@@ -291,7 +293,7 @@ impl client::Client<super::super::TokenStream, super::super::TokenStream> {
             run,
             force_show_panics,
         )
-        .map(<MarkedTypes<S> as Types>::TokenStream::unmark)
+        .map(|s| <Option<<MarkedTypes<S> as Types>::TokenStream>>::unmark(s).unwrap_or_default())
     }
 }
 
@@ -301,14 +303,18 @@ impl
         super::super::TokenStream,
     >
 {
-    pub fn run<S: Server>(
+    pub fn run<S>(
         &self,
         strategy: &impl ExecutionStrategy,
         server: S,
         input: S::TokenStream,
         input2: S::TokenStream,
         force_show_panics: bool,
-    ) -> Result<S::TokenStream, PanicMessage> {
+    ) -> Result<S::TokenStream, PanicMessage>
+    where
+        S: Server,
+        S::TokenStream: Default,
+    {
         let client::Client { get_handle_counters, run, _marker } = *self;
         run_server(
             strategy,
@@ -321,6 +327,6 @@ impl
             run,
             force_show_panics,
         )
-        .map(<MarkedTypes<S> as Types>::TokenStream::unmark)
+        .map(|s| <Option<<MarkedTypes<S> as Types>::TokenStream>>::unmark(s).unwrap_or_default())
     }
 }

crates/proc-macro-srv/src/abis/abi_1_63/proc_macro/mod.rs

@@ -47,7 +47,7 @@ pub fn is_available() -> bool {
 /// This is both the input and output of `#[proc_macro]`, `#[proc_macro_attribute]`
 /// and `#[proc_macro_derive]` definitions.
 #[derive(Clone)]
-pub struct TokenStream(bridge::client::TokenStream);
+pub struct TokenStream(Option<bridge::client::TokenStream>);
 
 /// Error returned from `TokenStream::from_str`.
 #[non_exhaustive]
@@ -78,12 +78,12 @@ impl error::Error for ExpandError {}
 impl TokenStream {
     /// Returns an empty `TokenStream` containing no token trees.
     pub fn new() -> TokenStream {
-        TokenStream(bridge::client::TokenStream::new())
+        TokenStream(None)
     }
 
     /// Checks if this `TokenStream` is empty.
     pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
+        self.0.as_ref().map(|h| h.is_empty()).unwrap_or(true)
     }
 
     /// Parses this `TokenStream` as an expression and attempts to expand any
@@ -97,8 +97,9 @@ impl TokenStream {
     /// specific behavior for any error condition, and what conditions are
     /// considered errors, is unspecified and may change in the future.
     pub fn expand_expr(&self) -> Result<TokenStream, ExpandError> {
-        match bridge::client::TokenStream::expand_expr(&self.0) {
+        let stream = self.0.as_ref().ok_or(ExpandError)?;
-            Ok(stream) => Ok(TokenStream(stream)),
+        match bridge::client::TokenStream::expand_expr(stream) {
+            Ok(stream) => Ok(TokenStream(Some(stream))),
             Err(_) => Err(ExpandError),
         }
     }
@@ -115,7 +116,7 @@ impl FromStr for TokenStream {
     type Err = LexError;
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
-        Ok(TokenStream(bridge::client::TokenStream::from_str(src)))
+        Ok(TokenStream(Some(bridge::client::TokenStream::from_str(src))))
     }
 }
 
@@ -144,15 +145,64 @@ impl Default for TokenStream {
 
 pub use quote::{quote, quote_span};
 
+fn tree_to_bridge_tree(
+    tree: TokenTree,
+) -> bridge::TokenTree<
+    bridge::client::Group,
+    bridge::client::Punct,
+    bridge::client::Ident,
+    bridge::client::Literal,
+> {
+    match tree {
+        TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0),
+        TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0),
+        TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0),
+        TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0),
+    }
+}
+
 /// Creates a token stream containing a single token tree.
 impl From<TokenTree> for TokenStream {
     fn from(tree: TokenTree) -> TokenStream {
-        TokenStream(bridge::client::TokenStream::from_token_tree(match tree {
+        TokenStream(Some(bridge::client::TokenStream::from_token_tree(tree_to_bridge_tree(tree))))
-            TokenTree::Group(tt) => bridge::TokenTree::Group(tt.0),
+    }
-            TokenTree::Punct(tt) => bridge::TokenTree::Punct(tt.0),
+}
-            TokenTree::Ident(tt) => bridge::TokenTree::Ident(tt.0),
+
-            TokenTree::Literal(tt) => bridge::TokenTree::Literal(tt.0),
+/// Non-generic helper for implementing `FromIterator<TokenStream>` and
-        }))
+/// `Extend<TokenStream>` with less monomorphization in calling crates.
+struct ConcatStreamsHelper {
+    streams: Vec<bridge::client::TokenStream>,
+}
+
+impl ConcatStreamsHelper {
+    fn new(capacity: usize) -> Self {
+        ConcatStreamsHelper { streams: Vec::with_capacity(capacity) }
+    }
+
+    fn push(&mut self, stream: TokenStream) {
+        if let Some(stream) = stream.0 {
+            self.streams.push(stream);
+        }
+    }
+
+    fn build(mut self) -> TokenStream {
+        if self.streams.len() <= 1 {
+            TokenStream(self.streams.pop())
+        } else {
+            TokenStream(Some(bridge::client::TokenStream::concat_streams(None, self.streams)))
+        }
+    }
+
+    fn append_to(mut self, stream: &mut TokenStream) {
+        if self.streams.is_empty() {
+            return;
+        }
+        let base = stream.0.take();
+        if base.is_none() && self.streams.len() == 1 {
+            stream.0 = self.streams.pop();
+        } else {
+            stream.0 = Some(bridge::client::TokenStream::concat_streams(base, self.streams));
+        }
     }
 }
 
@@ -167,9 +217,10 @@ impl iter::FromIterator<TokenTree> for TokenStream {
 /// from multiple token streams into a single stream.
 impl iter::FromIterator<TokenStream> for TokenStream {
     fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
-        let mut builder = bridge::client::TokenStreamBuilder::new();
+        let iter = streams.into_iter();
-        streams.into_iter().for_each(|stream| builder.push(stream.0));
+        let mut builder = ConcatStreamsHelper::new(iter.size_hint().0);
-        TokenStream(builder.build())
+        iter.for_each(|stream| builder.push(stream));
+        builder.build()
     }
 }
 
@@ -194,7 +245,16 @@ pub mod token_stream {
     /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups,
     /// and returns whole groups as token trees.
     #[derive(Clone)]
-    pub struct IntoIter(bridge::client::TokenStreamIter);
+    pub struct IntoIter(
+        std::vec::IntoIter<
+            bridge::TokenTree<
+                bridge::client::Group,
+                bridge::client::Punct,
+                bridge::client::Ident,
+                bridge::client::Literal,
+            >,
+        >,
+    );
 
     impl Iterator for IntoIter {
         type Item = TokenTree;
@@ -214,7 +274,7 @@ pub mod token_stream {
         type IntoIter = IntoIter;
 
         fn into_iter(self) -> IntoIter {
-            IntoIter(self.0.into_iter())
+            IntoIter(self.0.map(|v| v.into_trees()).unwrap_or_default().into_iter())
         }
     }
 }
@@ -560,7 +620,7 @@ impl Group {
     /// Note that the returned token stream does not include the delimiter
     /// returned above.
     pub fn stream(&self) -> TokenStream {
-        TokenStream(self.0.stream())
+        TokenStream(Some(self.0.stream()))
     }
 
     /// Returns the span for the delimiters of this token stream, spanning the

crates/proc-macro-srv/src/abis/abi_1_63/rustc_server.rs

@@ -23,14 +23,14 @@ type Spacing = tt::Spacing;
 type Literal = tt::Literal;
 type Span = tt::TokenId;
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Default, Clone)]
 pub struct TokenStream {
     pub token_trees: Vec<TokenTree>,
 }
 
 impl TokenStream {
     pub fn new() -> Self {
-        TokenStream { token_trees: Default::default() }
+        TokenStream::default()
     }
 
     pub fn with_subtree(subtree: tt::Subtree) -> Self {
@@ -276,8 +276,6 @@ pub struct Rustc {
 impl server::Types for Rustc {
     type FreeFunctions = FreeFunctions;
     type TokenStream = TokenStream;
-    type TokenStreamBuilder = TokenStreamBuilder;
-    type TokenStreamIter = TokenStreamIter;
     type Group = Group;
     type Punct = Punct;
     type Ident = IdentId;
@@ -297,10 +295,6 @@ impl server::FreeFunctions for Rustc {
 }
 
 impl server::TokenStream for Rustc {
-    fn new(&mut self) -> Self::TokenStream {
-        Self::TokenStream::new()
-    }
-
     fn is_empty(&mut self, stream: &Self::TokenStream) -> bool {
         stream.is_empty()
     }
@@ -344,41 +338,55 @@ impl server::TokenStream for Rustc {
         }
     }
 
-    fn into_iter(&mut self, stream: Self::TokenStream) -> Self::TokenStreamIter {
-        let trees: Vec<TokenTree> = stream.into_iter().collect();
-        TokenStreamIter { trees: trees.into_iter() }
-    }
-
     fn expand_expr(&mut self, self_: &Self::TokenStream) -> Result<Self::TokenStream, ()> {
         Ok(self_.clone())
     }
-}
 
-impl server::TokenStreamBuilder for Rustc {
+    fn concat_trees(
-    fn new(&mut self) -> Self::TokenStreamBuilder {
+        &mut self,
-        Self::TokenStreamBuilder::new()
+        base: Option<Self::TokenStream>,
-    }
+        trees: Vec<bridge::TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>>,
-    fn push(&mut self, builder: &mut Self::TokenStreamBuilder, stream: Self::TokenStream) {
+    ) -> Self::TokenStream {
-        builder.push(stream)
+        let mut builder = TokenStreamBuilder::new();
-    }
+        if let Some(base) = base {
-    fn build(&mut self, builder: Self::TokenStreamBuilder) -> Self::TokenStream {
+            builder.push(base);
+        }
+        for tree in trees {
+            builder.push(self.from_token_tree(tree));
+        }
         builder.build()
     }
-}
 
-impl server::TokenStreamIter for Rustc {
+    fn concat_streams(
-    fn next(
         &mut self,
-        iter: &mut Self::TokenStreamIter,
+        base: Option<Self::TokenStream>,
-    ) -> Option<bridge::TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
+        streams: Vec<Self::TokenStream>,
-        iter.trees.next().map(|tree| match tree {
+    ) -> Self::TokenStream {
-            TokenTree::Subtree(group) => bridge::TokenTree::Group(group),
+        let mut builder = TokenStreamBuilder::new();
-            TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
+        if let Some(base) = base {
-                bridge::TokenTree::Ident(IdentId(self.ident_interner.intern(&IdentData(ident))))
+            builder.push(base);
-            }
+        }
-            TokenTree::Leaf(tt::Leaf::Literal(literal)) => bridge::TokenTree::Literal(literal),
+        for stream in streams {
-            TokenTree::Leaf(tt::Leaf::Punct(punct)) => bridge::TokenTree::Punct(punct),
+            builder.push(stream);
-        })
+        }
+        builder.build()
+    }
+
+    fn into_trees(
+        &mut self,
+        stream: Self::TokenStream,
+    ) -> Vec<bridge::TokenTree<Self::Group, Self::Punct, Self::Ident, Self::Literal>> {
+        stream
+            .into_iter()
+            .map(|tree| match tree {
+                tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) => {
+                    bridge::TokenTree::Ident(IdentId(self.ident_interner.intern(&IdentData(ident))))
+                }
+                tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => bridge::TokenTree::Literal(lit),
+                tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) => bridge::TokenTree::Punct(punct),
+                tt::TokenTree::Subtree(subtree) => bridge::TokenTree::Group(subtree),
+            })
+            .collect()
     }
 }
 
@@ -416,8 +424,15 @@ fn spacing_to_external(spacing: Spacing) -> bridge::Spacing {
 }
 
 impl server::Group for Rustc {
-    fn new(&mut self, delimiter: bridge::Delimiter, stream: Self::TokenStream) -> Self::Group {
+    fn new(
-        Self::Group { delimiter: delim_to_internal(delimiter), token_trees: stream.token_trees }
+        &mut self,
+        delimiter: bridge::Delimiter,
+        stream: Option<Self::TokenStream>,
+    ) -> Self::Group {
+        Self::Group {
+            delimiter: delim_to_internal(delimiter),
+            token_trees: stream.unwrap_or_default().token_trees,
+        }
     }
     fn delimiter(&mut self, group: &Self::Group) -> bridge::Delimiter {
         delim_to_external(group.delimiter)