internal: rename

crates/mbe/src/lib.rs

@@ -10,7 +10,7 @@ mod parser;
 mod expander;
 mod syntax_bridge;
 mod tt_iter;
-mod to_parser_tokens;
+mod to_parser_input;
 
 #[cfg(test)]
 mod benchmark;

crates/mbe/src/syntax_bridge.rs

@@ -10,7 +10,7 @@ use syntax::{
 use tt::buffer::{Cursor, TokenBuffer};
 
 use crate::{
-    to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
+    to_parser_input::to_parser_input, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,
 };
 
 /// Convert the syntax node to a `TokenTree` (what macro
@@ -54,17 +54,17 @@ pub fn token_tree_to_syntax_node(
         }
         _ => TokenBuffer::from_subtree(tt),
     };
-    let parser_tokens = to_parser_tokens(&buffer);
+    let parser_input = to_parser_input(&buffer);
-    let tree_traversal = parser::parse(&parser_tokens, entry_point);
+    let parser_output = parser::parse(&parser_input, entry_point);
     let mut tree_sink = TtTreeSink::new(buffer.begin());
-    for event in tree_traversal.iter() {
+    for event in parser_output.iter() {
         match event {
-            parser::TraversalStep::Token { kind, n_raw_tokens } => {
+            parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
                 tree_sink.token(kind, n_raw_tokens)
             }
-            parser::TraversalStep::EnterNode { kind } => tree_sink.start_node(kind),
+            parser::Step::Enter { kind } => tree_sink.start_node(kind),
-            parser::TraversalStep::LeaveNode => tree_sink.finish_node(),
+            parser::Step::Exit => tree_sink.finish_node(),
-            parser::TraversalStep::Error { msg } => tree_sink.error(msg.to_string()),
+            parser::Step::Error { msg } => tree_sink.error(msg.to_string()),
         }
     }
     if tree_sink.roots.len() != 1 {

crates/mbe/src/to_parser_input.rs

@@ -4,8 +4,8 @@
 use syntax::{SyntaxKind, SyntaxKind::*, T};
 use tt::buffer::TokenBuffer;
 
-pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
+pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {
-    let mut res = parser::Tokens::default();
+    let mut res = parser::Input::default();
 
     let mut current = buffer.begin();
 

crates/mbe/src/tt_iter.rs

@@ -1,7 +1,7 @@
 //! A "Parser" structure for token trees. We use this when parsing a declarative
 //! macro definition into a list of patterns and templates.
 
-use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
+use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult, ParserEntryPoint};
 
 use syntax::SyntaxKind;
 use tt::buffer::TokenBuffer;
@@ -94,23 +94,23 @@ impl<'a> TtIter<'a> {
         entry_point: ParserEntryPoint,
     ) -> ExpandResult<Option<tt::TokenTree>> {
         let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
-        let parser_tokens = to_parser_tokens(&buffer);
+        let parser_input = to_parser_input(&buffer);
-        let tree_traversal = parser::parse(&parser_tokens, entry_point);
+        let tree_traversal = parser::parse(&parser_input, entry_point);
 
         let mut cursor = buffer.begin();
         let mut error = false;
         for step in tree_traversal.iter() {
             match step {
-                parser::TraversalStep::Token { kind, mut n_raw_tokens } => {
+                parser::Step::Token { kind, mut n_input_tokens } => {
                     if kind == SyntaxKind::LIFETIME_IDENT {
-                        n_raw_tokens = 2;
+                        n_input_tokens = 2;
                     }
-                    for _ in 0..n_raw_tokens {
+                    for _ in 0..n_input_tokens {
                         cursor = cursor.bump_subtree();
                     }
                 }
-                parser::TraversalStep::EnterNode { .. } | parser::TraversalStep::LeaveNode => (),
+                parser::Step::Enter { .. } | parser::Step::Exit => (),
-                parser::TraversalStep::Error { .. } => error = true,
+                parser::Step::Error { .. } => error = true,
             }
         }
 

crates/parser/src/event.rs

@@ -10,7 +10,7 @@
 use std::mem;
 
 use crate::{
-    tree_traversal::TreeTraversal,
+    output::Output,
     SyntaxKind::{self, *},
 };
 
@@ -87,8 +87,8 @@ impl Event {
 }
 
 /// Generate the syntax tree with the control of events.
-pub(super) fn process(mut events: Vec<Event>) -> TreeTraversal {
+pub(super) fn process(mut events: Vec<Event>) -> Output {
-    let mut res = TreeTraversal::default();
+    let mut res = Output::default();
     let mut forward_parents = Vec::new();
 
     for i in 0..events.len() {

crates/parser/src/input.rs

@@ -1,26 +1,26 @@
-//! Input for the parser -- a sequence of tokens.
+//! See [`Input`].
-//!
-//! As of now, parser doesn't have access to the *text* of the tokens, and makes
-//! decisions based solely on their classification. Unlike `LexerToken`, the
-//! `Tokens` doesn't include whitespace and comments.
 
 use crate::SyntaxKind;
 
 #[allow(non_camel_case_types)]
 type bits = u64;
 
-/// Main input to the parser.
+/// Input for the parser -- a sequence of tokens.
 ///
-/// A sequence of tokens represented internally as a struct of arrays.
+/// As of now, parser doesn't have access to the *text* of the tokens, and makes
+/// decisions based solely on their classification. Unlike `LexerToken`, the
+/// `Tokens` doesn't include whitespace and comments. Main input to the parser.
+///
+/// Struct of arrays internally, but this shouldn't really matter.
 #[derive(Default)]
-pub struct Tokens {
+pub struct Input {
     kind: Vec<SyntaxKind>,
     joint: Vec<bits>,
     contextual_kind: Vec<SyntaxKind>,
 }
 
 /// `pub` impl used by callers to create `Tokens`.
-impl Tokens {
+impl Input {
     #[inline]
     pub fn push(&mut self, kind: SyntaxKind) {
         self.push_impl(kind, SyntaxKind::EOF)
@@ -63,7 +63,7 @@ impl Tokens {
 }
 
 /// pub(crate) impl used by the parser to consume `Tokens`.
-impl Tokens {
+impl Input {
     pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
         self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
     }
@@ -76,7 +76,7 @@ impl Tokens {
     }
 }
 
-impl Tokens {
+impl Input {
     fn bit_index(&self, n: usize) -> (usize, usize) {
         let idx = n / (bits::BITS as usize);
         let b_idx = n % (bits::BITS as usize);

crates/parser/src/lexed_str.rs

@@ -122,8 +122,8 @@ impl<'a> LexedStr<'a> {
         self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))
     }
 
-    pub fn to_tokens(&self) -> crate::Tokens {
+    pub fn to_input(&self) -> crate::Input {
-        let mut res = crate::Tokens::default();
+        let mut res = crate::Input::default();
         let mut was_joint = false;
         for i in 0..self.len() {
             let kind = self.kind(i);

crates/parser/src/lib.rs

@@ -24,8 +24,8 @@ mod syntax_kind;
 mod event;
 mod parser;
 mod grammar;
-mod tokens;
+mod input;
-mod tree_traversal;
+mod output;
 
 #[cfg(test)]
 mod tests;
@@ -33,10 +33,10 @@ mod tests;
 pub(crate) use token_set::TokenSet;
 
 pub use crate::{
+    input::Input,
     lexed_str::LexedStr,
+    output::{Output, Step},
     syntax_kind::SyntaxKind,
-    tokens::Tokens,
-    tree_traversal::{TraversalStep, TreeTraversal},
 };
 
 /// rust-analyzer parser allows you to choose one of the possible entry points.
@@ -62,11 +62,19 @@ pub enum ParserEntryPoint {
 }
 
 /// Parse given tokens into the given sink as a rust file.
-pub fn parse_source_file(tokens: &Tokens) -> TreeTraversal {
+pub fn parse_source_file(inp: &Input) -> Output {
-    parse(tokens, ParserEntryPoint::SourceFile)
+    parse(inp, ParserEntryPoint::SourceFile)
 }
 
-pub fn parse(tokens: &Tokens, entry_point: ParserEntryPoint) -> TreeTraversal {
+/// Parses the given [`Input`] into [`Output`] assuming that the top-level
+/// syntactic construct is the given [`ParserEntryPoint`].
+///
+/// Both input and output here are fairly abstract. The overall flow is that the
+/// caller has some "real" tokens, converts them to [`Input`], parses them to
+/// [`Output`], and then converts that into a "real" tree. The "real" tree is
+/// made of "real" tokens, so this all hinges on rather tight coordination of
+/// indices between the four stages.
+pub fn parse(inp: &Input, entry_point: ParserEntryPoint) -> Output {
     let entry_point: fn(&'_ mut parser::Parser) = match entry_point {
         ParserEntryPoint::SourceFile => grammar::entry_points::source_file,
         ParserEntryPoint::Path => grammar::entry_points::path,
@@ -84,7 +92,7 @@ pub fn parse(tokens: &Tokens, entry_point: ParserEntryPoint) -> TreeTraversal {
         ParserEntryPoint::Attr => grammar::entry_points::attr,
     };
 
-    let mut p = parser::Parser::new(tokens);
+    let mut p = parser::Parser::new(inp);
     entry_point(&mut p);
     let events = p.finish();
     event::process(events)
@@ -107,7 +115,7 @@ impl Reparser {
     ///
     /// Tokens must start with `{`, end with `}` and form a valid brace
     /// sequence.
-    pub fn parse(self, tokens: &Tokens) -> TreeTraversal {
+    pub fn parse(self, tokens: &Input) -> Output {
         let Reparser(r) = self;
         let mut p = parser::Parser::new(tokens);
         r(&mut p);

crates/parser/src/output.rs

@@ -1,43 +1,52 @@
-//! TODO
+//! See [`Output`]
+
 use crate::SyntaxKind;
 
-/// Output of the parser.
+/// Output of the parser -- a DFS traversal of a concrete syntax tree.
+///
+/// Use the [`Output::iter`] method to iterate over traversal steps and consume
+/// a syntax tree.
+///
+/// In a sense, this is just a sequence of [`SyntaxKind`]-colored parenthesis
+/// interspersed into the original [`crate::Input`]. The output is fundamentally
+/// coordinated with the input and `n_input_tokens` refers to the number of
+/// times [`crate::Input::push`] was called.
 #[derive(Default)]
-pub struct TreeTraversal {
+pub struct Output {
     /// 32-bit encoding of events. If LSB is zero, then that's an index into the
     /// error vector. Otherwise, it's one of the thee other variants, with data encoded as
     ///
-    ///     |16 bit kind|8 bit n_raw_tokens|4 bit tag|4 bit leftover|
+    ///     |16 bit kind|8 bit n_input_tokens|4 bit tag|4 bit leftover|
     ///
     event: Vec<u32>,
     error: Vec<String>,
 }
 
-pub enum TraversalStep<'a> {
+pub enum Step<'a> {
-    Token { kind: SyntaxKind, n_raw_tokens: u8 },
+    Token { kind: SyntaxKind, n_input_tokens: u8 },
-    EnterNode { kind: SyntaxKind },
+    Enter { kind: SyntaxKind },
-    LeaveNode,
+    Exit,
     Error { msg: &'a str },
 }
 
-impl TreeTraversal {
+impl Output {
-    pub fn iter(&self) -> impl Iterator<Item = TraversalStep<'_>> {
+    pub fn iter(&self) -> impl Iterator<Item = Step<'_>> {
         self.event.iter().map(|&event| {
             if event & 0b1 == 0 {
-                return TraversalStep::Error { msg: self.error[(event as usize) >> 1].as_str() };
+                return Step::Error { msg: self.error[(event as usize) >> 1].as_str() };
             }
             let tag = ((event & 0x0000_00F0) >> 4) as u8;
             match tag {
                 0 => {
                     let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
-                    let n_raw_tokens = ((event & 0x0000_FF00) >> 8) as u8;
+                    let n_input_tokens = ((event & 0x0000_FF00) >> 8) as u8;
-                    TraversalStep::Token { kind, n_raw_tokens }
+                    Step::Token { kind, n_input_tokens }
                 }
                 1 => {
                     let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
-                    TraversalStep::EnterNode { kind }
+                    Step::Enter { kind }
                 }
-                2 => TraversalStep::LeaveNode,
+                2 => Step::Exit,
                 _ => unreachable!(),
             }
         })

crates/parser/src/parser.rs

@@ -7,7 +7,7 @@ use limit::Limit;
 
 use crate::{
     event::Event,
-    tokens::Tokens,
+    input::Input,
     SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
     TokenSet, T,
 };
@@ -22,7 +22,7 @@ use crate::{
 /// "start expression, consume number literal,
 /// finish expression". See `Event` docs for more.
 pub(crate) struct Parser<'t> {
-    tokens: &'t Tokens,
+    inp: &'t Input,
     pos: usize,
     events: Vec<Event>,
     steps: Cell<u32>,
@@ -31,8 +31,8 @@ pub(crate) struct Parser<'t> {
 static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);
 
 impl<'t> Parser<'t> {
-    pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> {
+    pub(super) fn new(inp: &'t Input) -> Parser<'t> {
-        Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) }
+        Parser { inp, pos: 0, events: Vec::new(), steps: Cell::new(0) }
     }
 
     pub(crate) fn finish(self) -> Vec<Event> {
@@ -55,7 +55,7 @@ impl<'t> Parser<'t> {
         assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");
         self.steps.set(steps + 1);
 
-        self.tokens.kind(self.pos + n)
+        self.inp.kind(self.pos + n)
     }
 
     /// Checks if the current token is `kind`.
@@ -91,7 +91,7 @@ impl<'t> Parser<'t> {
             T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
             T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
 
-            _ => self.tokens.kind(self.pos + n) == kind,
+            _ => self.inp.kind(self.pos + n) == kind,
         }
     }
 
@@ -130,17 +130,17 @@ impl<'t> Parser<'t> {
     }
 
     fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
-        self.tokens.kind(self.pos + n) == k1
+        self.inp.kind(self.pos + n) == k1
-            && self.tokens.kind(self.pos + n + 1) == k2
+            && self.inp.kind(self.pos + n + 1) == k2
-            && self.tokens.is_joint(self.pos + n)
+            && self.inp.is_joint(self.pos + n)
     }
 
     fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
-        self.tokens.kind(self.pos + n) == k1
+        self.inp.kind(self.pos + n) == k1
-            && self.tokens.kind(self.pos + n + 1) == k2
+            && self.inp.kind(self.pos + n + 1) == k2
-            && self.tokens.kind(self.pos + n + 2) == k3
+            && self.inp.kind(self.pos + n + 2) == k3
-            && self.tokens.is_joint(self.pos + n)
+            && self.inp.is_joint(self.pos + n)
-            && self.tokens.is_joint(self.pos + n + 1)
+            && self.inp.is_joint(self.pos + n + 1)
     }
 
     /// Checks if the current token is in `kinds`.
@@ -150,7 +150,7 @@ impl<'t> Parser<'t> {
 
     /// Checks if the current token is contextual keyword with text `t`.
     pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {
-        self.tokens.contextual_kind(self.pos) == kw
+        self.inp.contextual_kind(self.pos) == kw
     }
 
     /// Starts a new node in the syntax tree. All nodes and tokens

crates/syntax/src/parsing.rs

@@ -12,9 +12,9 @@ pub(crate) use crate::parsing::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
     let lexed = parser::LexedStr::new(text);
-    let parser_tokens = lexed.to_tokens();
+    let parser_input = lexed.to_input();
-    let tree_traversal = parser::parse_source_file(&parser_tokens);
+    let parser_output = parser::parse_source_file(&parser_input);
-    let (node, errors, _eof) = build_tree(lexed, tree_traversal, false);
+    let (node, errors, _eof) = build_tree(lexed, parser_output, false);
     (node, errors)
 }
 
@@ -27,9 +27,9 @@ pub(crate) fn parse_text_as<T: AstNode>(
     if lexed.errors().next().is_some() {
         return Err(());
     }
-    let parser_tokens = lexed.to_tokens();
+    let parser_input = lexed.to_input();
-    let tree_traversal = parser::parse(&parser_tokens, entry_point);
+    let parser_output = parser::parse(&parser_input, entry_point);
-    let (node, errors, eof) = build_tree(lexed, tree_traversal, true);
+    let (node, errors, eof) = build_tree(lexed, parser_output, true);
 
     if !errors.is_empty() || !eof {
         return Err(());

crates/syntax/src/parsing/reparsing.rs

@@ -89,7 +89,7 @@ fn reparse_block(
     let text = get_text_after_edit(node.clone().into(), edit);
 
     let lexed = parser::LexedStr::new(text.as_str());
-    let parser_tokens = lexed.to_tokens();
+    let parser_input = lexed.to_input();
     if !is_balanced(&lexed) {
         return None;
     }

crates/syntax/src/parsing/text_tree_sink.rs

@@ -2,7 +2,7 @@
 
 use std::mem;
 
-use parser::{LexedStr, TreeTraversal};
+use parser::LexedStr;
 
 use crate::{
     ast,
@@ -14,7 +14,7 @@ use crate::{
 
 pub(crate) fn build_tree(
     lexed: LexedStr<'_>,
-    tree_traversal: TreeTraversal,
+    parser_output: parser::Output,
     synthetic_root: bool,
 ) -> (GreenNode, Vec<SyntaxError>, bool) {
     let mut builder = TextTreeSink::new(lexed);
@@ -23,14 +23,14 @@ pub(crate) fn build_tree(
         builder.start_node(SyntaxKind::SOURCE_FILE);
     }
 
-    for event in tree_traversal.iter() {
+    for event in parser_output.iter() {
         match event {
-            parser::TraversalStep::Token { kind, n_raw_tokens } => {
+            parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {
                 builder.token(kind, n_raw_tokens)
             }
-            parser::TraversalStep::EnterNode { kind } => builder.start_node(kind),
+            parser::Step::Enter { kind } => builder.start_node(kind),
-            parser::TraversalStep::LeaveNode => builder.finish_node(),
+            parser::Step::Exit => builder.finish_node(),
-            parser::TraversalStep::Error { msg } => {
+            parser::Step::Error { msg } => {
                 let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap();
                 builder.inner.error(msg.to_string(), text_pos);
             }
@@ -45,7 +45,7 @@ pub(crate) fn build_tree(
 /// Bridges the parser with our specific syntax tree representation.
 ///
 /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
-pub(crate) struct TextTreeSink<'a> {
+struct TextTreeSink<'a> {
     lexed: LexedStr<'a>,
     pos: usize,
     state: State,