//! TokenStream implementation used by sysroot ABI use tt::TokenTree; #[derive(Debug, Clone)] pub struct TokenStream { pub(super) token_trees: Vec>, } impl Default for TokenStream { fn default() -> Self { Self { token_trees: vec![] } } } impl TokenStream { pub(crate) fn new() -> Self { TokenStream::default() } pub(crate) fn with_subtree(subtree: tt::Subtree) -> Self { if subtree.delimiter.kind != tt::DelimiterKind::Invisible { TokenStream { token_trees: vec![TokenTree::Subtree(subtree)] } } else { TokenStream { token_trees: subtree.token_trees.into_vec() } } } pub(crate) fn into_subtree(self, call_site: S) -> tt::Subtree where S: Copy, { tt::Subtree { delimiter: tt::Delimiter { open: call_site, close: call_site, kind: tt::DelimiterKind::Invisible, }, token_trees: self.token_trees.into_boxed_slice(), } } pub(super) fn is_empty(&self) -> bool { self.token_trees.is_empty() } } /// Creates a token stream containing a single token tree. impl From> for TokenStream { fn from(tree: TokenTree) -> TokenStream { TokenStream { token_trees: vec![tree] } } } /// Collects a number of token trees into a single stream. impl FromIterator> for TokenStream { fn from_iter>>(trees: I) -> Self { trees.into_iter().map(TokenStream::from).collect() } } /// A "flattening" operation on token streams, collects token trees /// from multiple token streams into a single stream. impl FromIterator> for TokenStream { fn from_iter>>(streams: I) -> Self { let mut builder = TokenStreamBuilder::new(); streams.into_iter().for_each(|stream| builder.push(stream)); builder.build() } } impl Extend> for TokenStream { fn extend>>(&mut self, trees: I) { self.extend(trees.into_iter().map(TokenStream::from)); } } impl Extend> for TokenStream { fn extend>>(&mut self, streams: I) { for item in streams { for tkn in item { match tkn { tt::TokenTree::Subtree(subtree) if subtree.delimiter.kind == tt::DelimiterKind::Invisible => { self.token_trees.extend(subtree.token_trees.into_vec().into_iter()); } _ => { self.token_trees.push(tkn); } } } } } } pub(super) struct TokenStreamBuilder { acc: TokenStream, } /// pub(super)lic implementation details for the `TokenStream` type, such as iterators. pub(super) mod token_stream { use super::{TokenStream, TokenTree}; /// An iterator over `TokenStream`'s `TokenTree`s. /// The iteration is "shallow", e.g., the iterator doesn't recurse into delimited groups, /// and returns whole groups as token trees. impl IntoIterator for TokenStream { type Item = TokenTree; type IntoIter = std::vec::IntoIter>; fn into_iter(self) -> Self::IntoIter { self.token_trees.into_iter() } } type LexError = String; /// Attempts to break the string into tokens and parse those tokens into a token stream. /// May fail for a number of reasons, for example, if the string contains unbalanced delimiters /// or characters not existing in the language. /// All tokens in the parsed stream get `Span::call_site()` spans. /// /// NOTE: some errors may cause panics instead of returning `LexError`. We reserve the right to /// change these errors into `LexError`s later. #[rustfmt::skip] impl /*FromStr for*/ TokenStream { // type Err = LexError; pub(crate) fn from_str(src: &str, call_site: S) -> Result, LexError> { let subtree = mbe::parse_to_token_tree_static_span(call_site, src).ok_or("Failed to parse from mbe")?; Ok(TokenStream::with_subtree(subtree)) } } impl ToString for TokenStream { fn to_string(&self) -> String { ::tt::pretty(&self.token_trees) } } } impl TokenStreamBuilder { pub(super) fn new() -> TokenStreamBuilder { TokenStreamBuilder { acc: TokenStream::new() } } pub(super) fn push(&mut self, stream: TokenStream) { self.acc.extend(stream.into_iter()) } pub(super) fn build(self) -> TokenStream { self.acc } }