internal: replace TreeSink with a data structure

The general theme of this is to make parser a better independent
library.

The specific thing we do here is replacing callback based TreeSink with
a data structure. That is, rather than calling user-provided tree
construction methods, the parser now spits out a very bare-bones tree,
effectively a log of a DFS traversal.

This makes the parser usable without any *specifc* tree sink, and allows
us to, eg, move tests into this crate.

Now, it's also true that this is a distinction without a difference, as
the old and the new interface are equivalent in expressiveness. Still,
this new thing seems somewhat simpler. But yeah, I admit I don't have a
suuper strong motivation here, just a hunch that this is better.

crates/syntax/src/parsing.rs

@@ -4,24 +4,18 @@
 mod text_tree_sink;
 mod reparsing;
 
-use parser::SyntaxKind;
-use text_tree_sink::TextTreeSink;
-
-use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode};
+use crate::{
+    parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode,
+};
 
 pub(crate) use crate::parsing::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
     let lexed = parser::LexedStr::new(text);
     let parser_tokens = lexed.to_tokens();
-
-    let mut tree_sink = TextTreeSink::new(lexed);
-
-    parser::parse_source_file(&parser_tokens, &mut tree_sink);
-
-    let (tree, parser_errors) = tree_sink.finish();
-
-    (tree, parser_errors)
+    let tree_traversal = parser::parse_source_file(&parser_tokens);
+    let (node, errors, _eof) = build_tree(lexed, tree_traversal, false);
+    (node, errors)
 }
 
 /// Returns `text` parsed as a `T` provided there are no parse errors.
@@ -34,20 +28,12 @@ pub(crate) fn parse_text_as<T: AstNode>(
         return Err(());
     }
     let parser_tokens = lexed.to_tokens();
+    let tree_traversal = parser::parse(&parser_tokens, entry_point);
+    let (node, errors, eof) = build_tree(lexed, tree_traversal, true);
 
-    let mut tree_sink = TextTreeSink::new(lexed);
-
-    // TextTreeSink assumes that there's at least some root node to which it can attach errors and
-    // tokens. We arbitrarily give it a SourceFile.
-    use parser::TreeSink;
-    tree_sink.start_node(SyntaxKind::SOURCE_FILE);
-    parser::parse(&parser_tokens, &mut tree_sink, entry_point);
-    tree_sink.finish_node();
-
-    let (tree, parser_errors, eof) = tree_sink.finish_eof();
-    if !parser_errors.is_empty() || !eof {
+    if !errors.is_empty() || !eof {
         return Err(());
     }
 
-    SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(())
+    SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(())
 }

crates/syntax/src/parsing/reparsing.rs

@@ -10,7 +10,7 @@ use parser::Reparser;
 use text_edit::Indel;
 
 use crate::{
-    parsing::text_tree_sink::TextTreeSink,
+    parsing::text_tree_sink::build_tree,
     syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},
     SyntaxError,
     SyntaxKind::*,
@@ -94,11 +94,9 @@ fn reparse_block(
         return None;
     }
 
-    let mut tree_sink = TextTreeSink::new(lexed);
+    let tree_traversal = reparser.parse(&parser_tokens);
 
-    reparser.parse(&parser_tokens, &mut tree_sink);
-
-    let (green, new_parser_errors) = tree_sink.finish();
+    let (green, new_parser_errors, _eof) = build_tree(lexed, tree_traversal, false);
 
     Some((node.replace_with(green), new_parser_errors, node.text_range()))
 }

crates/syntax/src/parsing/text_tree_sink.rs

@@ -2,7 +2,7 @@
 
 use std::mem;
 
-use parser::{LexedStr, ParseError, TreeSink};
+use parser::{LexedStr, TreeTraversal};
 
 use crate::{
     ast,
@@ -12,6 +12,36 @@ use crate::{
     SyntaxTreeBuilder, TextRange,
 };
 
+pub(crate) fn build_tree(
+    lexed: LexedStr<'_>,
+    tree_traversal: TreeTraversal,
+    synthetic_root: bool,
+) -> (GreenNode, Vec<SyntaxError>, bool) {
+    let mut builder = TextTreeSink::new(lexed);
+
+    if synthetic_root {
+        builder.start_node(SyntaxKind::SOURCE_FILE);
+    }
+
+    for event in tree_traversal.iter() {
+        match event {
+            parser::TraversalStep::Token { kind, n_raw_tokens } => {
+                builder.token(kind, n_raw_tokens)
+            }
+            parser::TraversalStep::EnterNode { kind } => builder.start_node(kind),
+            parser::TraversalStep::LeaveNode => builder.finish_node(),
+            parser::TraversalStep::Error { msg } => {
+                let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap();
+                builder.inner.error(msg.to_string(), text_pos);
+            }
+        }
+    }
+    if synthetic_root {
+        builder.finish_node()
+    }
+    builder.finish_eof()
+}
+
 /// Bridges the parser with our specific syntax tree representation.
 ///
 /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.
@@ -28,7 +58,7 @@ enum State {
     PendingFinish,
 }
 
-impl<'a> TreeSink for TextTreeSink<'a> {
+impl<'a> TextTreeSink<'a> {
     fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingStart => unreachable!(),
@@ -70,11 +100,6 @@ impl<'a> TreeSink for TextTreeSink<'a> {
             State::Normal => (),
         }
     }
-
-    fn error(&mut self, error: ParseError) {
-        let text_pos = self.lexed.text_start(self.pos).try_into().unwrap();
-        self.inner.error(error, text_pos);
-    }
 }
 
 impl<'a> TextTreeSink<'a> {
@@ -106,11 +131,6 @@ impl<'a> TextTreeSink<'a> {
         (node, errors, is_eof)
     }
 
-    pub(super) fn finish(self) -> (GreenNode, Vec<SyntaxError>) {
-        let (node, errors, _eof) = self.finish_eof();
-        (node, errors)
-    }
-
     fn eat_trivias(&mut self) {
         while self.pos < self.lexed.len() {
             let kind = self.lexed.kind(self.pos);

crates/syntax/src/syntax_node.rs

@@ -69,7 +69,7 @@ impl SyntaxTreeBuilder {
         self.inner.finish_node();
     }
 
-    pub fn error(&mut self, error: parser::ParseError, text_pos: TextSize) {
-        self.errors.push(SyntaxError::new_at_offset(*error.0, text_pos));
+    pub fn error(&mut self, error: String, text_pos: TextSize) {
+        self.errors.push(SyntaxError::new_at_offset(error, text_pos));
     }
 }