Merge #867

867: This moves the parser to separate crate r=matklad a=matklad

That makes parser independent form both the token and the tree representation. 

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

Cargo.lock

@@ -1071,6 +1071,13 @@ dependencies = [
  "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "ra_parser"
+version = "0.1.0"
+dependencies = [
+ "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "ra_project_model"
 version = "0.1.0"
@@ -1093,6 +1100,7 @@ dependencies = [
  "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ra_parser 0.1.0",
  "ra_text_edit 0.1.0",
  "rowan 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "smol_str 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",

crates/ra_parser/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+edition = "2018"
+name = "ra_parser"
+version = "0.1.0"
+authors = ["rust-analyzer developers"]
+publish = false
+
+[dependencies]
+drop_bomb = "0.1.4"

crates/ra_parser/src/event.rs

@@ -10,8 +10,8 @@
 use std::mem;
 
 use crate::{
+    ParseError, TreeSink,
     SyntaxKind::{self, *},
-    parsing::{ParseError, TreeSink},
 };
 
 /// `Parser` produces a flat list of `Event`s.
@@ -84,7 +84,7 @@ impl Event {
 }
 
 /// Generate the syntax tree with the control of events.
-pub(super) fn process(sink: &mut impl TreeSink, mut events: Vec<Event>) {
+pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {
     let mut forward_parents = Vec::new();
 
     for i in 0..events.len() {

crates/ra_parser/src/grammar.rs

@@ -38,20 +38,18 @@ mod types;
 
 use crate::{
     SyntaxKind::{self, *},
-    parsing::{
+    TokenSet,
-        token_set::TokenSet,
+    parser::{CompletedMarker, Marker, Parser},
-        parser::{CompletedMarker, Marker, Parser}
-    },
 };
 
-pub(super) fn root(p: &mut Parser) {
+pub(crate) fn root(p: &mut Parser) {
     let m = p.start();
     p.eat(SHEBANG);
     items::mod_contents(p, false);
     m.complete(p, SOURCE_FILE);
 }
 
-pub(super) fn reparser(
+pub(crate) fn reparser(
     node: SyntaxKind,
     first_child: Option<SyntaxKind>,
     parent: Option<SyntaxKind>,

crates/ra_parser/src/lib.rs

@@ -0,0 +1,64 @@
+#[macro_use]
+mod token_set;
+mod syntax_kind;
+mod event;
+mod parser;
+mod grammar;
+
+pub(crate) use token_set::TokenSet;
+
+pub use syntax_kind::SyntaxKind;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct ParseError(pub String);
+
+/// `TreeSink` abstracts details of a particular syntax tree implementation.
+pub trait TreeSink {
+    /// Adds new leaf to the current branch.
+    fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8);
+
+    /// Start new branch and make it current.
+    fn start_branch(&mut self, kind: SyntaxKind, root: bool);
+
+    /// Finish current branch and restore previous
+    /// branch as current.
+    fn finish_branch(&mut self, root: bool);
+
+    fn error(&mut self, error: ParseError);
+}
+
+/// `TokenSource` abstracts the source of the tokens parser operates one.
+///
+/// Hopefully this will allow us to treat text and token trees in the same way!
+pub trait TokenSource {
+    fn token_kind(&self, pos: usize) -> SyntaxKind;
+    fn is_token_joint_to_next(&self, pos: usize) -> bool;
+    fn is_keyword(&self, pos: usize, kw: &str) -> bool;
+}
+
+pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+    let mut p = parser::Parser::new(token_source);
+    grammar::root(&mut p);
+    let events = p.finish();
+    event::process(tree_sink, events);
+}
+
+pub struct Reparser(fn(&mut parser::Parser));
+
+impl Reparser {
+    pub fn for_node(
+        node: SyntaxKind,
+        first_child: Option<SyntaxKind>,
+        parent: Option<SyntaxKind>,
+    ) -> Option<Reparser> {
+        grammar::reparser(node, first_child, parent).map(Reparser)
+    }
+
+    pub fn parse(self, token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) {
+        let Reparser(r) = self;
+        let mut p = parser::Parser::new(token_source);
+        r(&mut p);
+        let events = p.finish();
+        event::process(tree_sink, events);
+    }
+}

crates/ra_parser/src/parser.rs

@@ -4,11 +4,8 @@ use drop_bomb::DropBomb;
 
 use crate::{
     SyntaxKind::{self, ERROR, EOF, TOMBSTONE},
-    parsing::{
+    TokenSource, ParseError, TokenSet,
-        TokenSource, ParseError,
+    event::Event,
-        token_set::TokenSet,
-        event::Event,
-    },
 };
 
 /// `Parser` struct provides the low-level API for

crates/ra_parser/src/syntax_kind.rs

@@ -2,8 +2,6 @@ mod generated;
 
 use std::fmt;
 
-use crate::SyntaxKind::*;
-
 pub use self::generated::SyntaxKind;
 
 impl fmt::Debug for SyntaxKind {
@@ -20,7 +18,7 @@ pub(crate) struct SyntaxInfo {
 impl SyntaxKind {
     pub fn is_trivia(self) -> bool {
         match self {
-            WHITESPACE | COMMENT => true,
+            SyntaxKind::WHITESPACE | SyntaxKind::COMMENT => true,
             _ => false,
         }
     }

crates/ra_parser/src/syntax_kind/generated.rs

@@ -568,7 +568,7 @@ impl SyntaxKind {
             EOF => &SyntaxInfo { name: "EOF" },
         }
     }
-    pub(crate) fn from_keyword(ident: &str) -> Option<SyntaxKind> {
+    pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
         let kw = match ident {
             "use" => USE_KW,
             "fn" => FN_KW,
@@ -610,7 +610,7 @@ impl SyntaxKind {
         Some(kw)
     }
 
-    pub(crate) fn from_char(c: char) -> Option<SyntaxKind> {
+    pub fn from_char(c: char) -> Option<SyntaxKind> {
         let tok = match c {
             ';' => SEMI,
             ',' => COMMA,

crates/ra_parser/src/syntax_kind/generated.rs.tera

@@ -74,7 +74,7 @@ impl SyntaxKind {
             EOF => &SyntaxInfo { name: "EOF" },
         }
     }
-    pub(crate) fn from_keyword(ident: &str) -> Option<SyntaxKind> {
+    pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
         let kw = match ident {
 {%- for kw in keywords %}
             "{{kw}}" => {{kw | upper}}_KW,
@@ -84,7 +84,7 @@ impl SyntaxKind {
         Some(kw)
     }
 
-    pub(crate) fn from_char(c: char) -> Option<SyntaxKind> {
+    pub fn from_char(c: char) -> Option<SyntaxKind> {
         let tok = match c {
 {%- for t in single_byte_tokens %}
             '{{t.0}}' => {{t.1}},

crates/ra_syntax/Cargo.toml

@@ -21,6 +21,7 @@ text_unit = { version = "0.1.6", features = ["serde"] }
 smol_str = { version = "0.1.9", features = ["serde"] }
 
 ra_text_edit = { path = "../ra_text_edit" }
+ra_parser = { path = "../ra_parser" }
 
 [dev-dependencies]
 test_utils = { path = "../test_utils" }

crates/ra_syntax/src/lib.rs

@@ -16,7 +16,6 @@
 #![allow(missing_docs)]
 //#![warn(unreachable_pub)] // rust-lang/rust#47816
 
-mod syntax_kinds;
 mod syntax_node;
 mod syntax_text;
 mod syntax_error;
@@ -31,9 +30,9 @@ pub mod ast;
 pub mod utils;
 
 pub use rowan::{SmolStr, TextRange, TextUnit};
+pub use ra_parser::SyntaxKind;
 pub use crate::{
     ast::AstNode,
-    syntax_kinds::SyntaxKind,
     syntax_error::{SyntaxError, SyntaxErrorKind, Location},
     syntax_text::SyntaxText,
     syntax_node::{Direction,  SyntaxNode, WalkEvent, TreeArc},

crates/ra_syntax/src/parsing.rs

@@ -1,50 +1,28 @@
-#[macro_use]
-mod token_set;
 mod builder;
 mod lexer;
-mod event;
 mod input;
-mod parser;
-mod grammar;
 mod reparsing;
 
+use ra_parser::{parse, ParseError};
+
 use crate::{
     SyntaxKind, SyntaxError,
     parsing::{
         builder::TreeBuilder,
         input::ParserInput,
-        event::process,
-        parser::Parser,
     },
     syntax_node::GreenNode,
 };
 
 pub use self::lexer::{tokenize, Token};
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct ParseError(pub String);
-
 pub(crate) use self::reparsing::incremental_reparse;
 
 pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {
     let tokens = tokenize(&text);
-    let tree_sink = TreeBuilder::new(text, &tokens);
+    let token_source = ParserInput::new(text, &tokens);
-    parse_with(tree_sink, text, &tokens, grammar::root)
+    let mut tree_sink = TreeBuilder::new(text, &tokens);
-}
+    parse(&token_source, &mut tree_sink);
-
-fn parse_with<S: TreeSink>(
-    mut tree_sink: S,
-    text: &str,
-    tokens: &[Token],
-    f: fn(&mut Parser),
-) -> S::Tree {
-    let events = {
-        let input = ParserInput::new(text, &tokens);
-        let mut p = Parser::new(&input);
-        f(&mut p);
-        p.finish()
-    };
-    process(&mut tree_sink, events);
     tree_sink.finish()
 }
 

crates/ra_syntax/src/parsing/builder.rs

@@ -1,7 +1,9 @@
+use ra_parser::{TreeSink, ParseError};
+
 use crate::{
     SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange,
     SyntaxKind::{self, *},
-    parsing::{TreeSink, ParseError, Token},
+    parsing::Token,
     syntax_node::{GreenNode, RaTypes},
 };
 
@@ -17,8 +19,6 @@ pub(crate) struct TreeBuilder<'a> {
 }
 
 impl<'a> TreeSink for TreeBuilder<'a> {
-    type Tree = (GreenNode, Vec<SyntaxError>);
-
     fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) {
         self.eat_trivias();
         let n_tokens = n_tokens as usize;
@@ -65,10 +65,6 @@ impl<'a> TreeSink for TreeBuilder<'a> {
         let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos);
         self.errors.push(error)
     }
-
-    fn finish(self) -> (GreenNode, Vec<SyntaxError>) {
-        (self.inner.finish(), self.errors)
-    }
 }
 
 impl<'a> TreeBuilder<'a> {
@@ -82,6 +78,11 @@ impl<'a> TreeBuilder<'a> {
             inner: GreenNodeBuilder::new(),
         }
     }
+
+    pub(super) fn finish(self) -> (GreenNode, Vec<SyntaxError>) {
+        (self.inner.finish(), self.errors)
+    }
+
     fn eat_trivias(&mut self) {
         while let Some(&token) = self.tokens.get(self.token_pos) {
             if !token.kind.is_trivia() {

crates/ra_syntax/src/parsing/input.rs

@@ -1,9 +1,8 @@
+use ra_parser::TokenSource;
+
 use crate::{
     SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit,
-    parsing::{
+    parsing::lexer::Token,
-        TokenSource,
-        lexer::Token,
-    },
 };
 
 impl<'t> TokenSource for ParserInput<'t> {

crates/ra_syntax/src/parsing/reparsing.rs

@@ -1,18 +1,18 @@
+use ra_text_edit::AtomTextEdit;
+use ra_parser::Reparser;
+
 use crate::{
     SyntaxKind::*, TextRange, TextUnit,
     algo,
     syntax_node::{GreenNode, SyntaxNode},
     syntax_error::SyntaxError,
     parsing::{
-        grammar, parse_with,
+        input::ParserInput,
         builder::TreeBuilder,
-        parser::Parser,
         lexer::{tokenize, Token},
     }
 };
 
-use ra_text_edit::AtomTextEdit;
-
 pub(crate) fn incremental_reparse(
     node: &SyntaxNode,
     edit: &AtomTextEdit,
@@ -61,8 +61,10 @@ fn reparse_block<'node>(
     if !is_balanced(&tokens) {
         return None;
     }
-    let tree_sink = TreeBuilder::new(&text, &tokens);
+    let token_source = ParserInput::new(&text, &tokens);
-    let (green, new_errors) = parse_with(tree_sink, &text, &tokens, reparser);
+    let mut tree_sink = TreeBuilder::new(&text, &tokens);
+    reparser.parse(&token_source, &mut tree_sink);
+    let (green, new_errors) = tree_sink.finish();
     Some((node, green, new_errors))
 }
 
@@ -78,15 +80,12 @@ fn is_contextual_kw(text: &str) -> bool {
     }
 }
 
-fn find_reparsable_node(
+fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, Reparser)> {
-    node: &SyntaxNode,
-    range: TextRange,
-) -> Option<(&SyntaxNode, fn(&mut Parser))> {
     let node = algo::find_covering_node(node, range);
     node.ancestors().find_map(|node| {
         let first_child = node.first_child().map(|it| it.kind());
         let parent = node.parent().map(|it| it.kind());
-        grammar::reparser(node.kind(), first_child, parent).map(|r| (node, r))
+        Reparser::for_node(node.kind(), first_child, parent).map(|r| (node, r))
     })
 }
 

crates/ra_syntax/src/syntax_error.rs

@@ -1,6 +1,8 @@
 use std::fmt;
 
-use crate::{TextRange, TextUnit, parsing::ParseError};
+use ra_parser::ParseError;
+
+use crate::{TextRange, TextUnit};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct SyntaxError {

crates/tools/src/lib.rs

@@ -14,11 +14,11 @@ pub use teraron::{Mode, Overwrite, Verify};
 pub type Result<T> = std::result::Result<T, failure::Error>;
 
 pub const GRAMMAR: &str = "crates/ra_syntax/src/grammar.ron";
-const GRAMMAR_DIR: &str = "crates/ra_syntax/src/parsing/grammar";
+const GRAMMAR_DIR: &str = "crates/ra_parser/src/grammar";
 const OK_INLINE_TESTS_DIR: &str = "crates/ra_syntax/tests/data/parser/inline/ok";
 const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/tests/data/parser/inline/err";
 
-pub const SYNTAX_KINDS: &str = "crates/ra_syntax/src/syntax_kinds/generated.rs.tera";
+pub const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs.tera";
 pub const AST: &str = "crates/ra_syntax/src/ast/generated.rs.tera";
 const TOOLCHAIN: &str = "stable";