diff --git a/Cargo.lock b/Cargo.lock index 7b1412556c..f09cf98c88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1071,6 +1071,13 @@ dependencies = [ "rustc-hash 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ra_parser" +version = "0.1.0" +dependencies = [ + "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ra_project_model" version = "0.1.0" @@ -1093,6 +1100,7 @@ dependencies = [ "drop_bomb 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ra_parser 0.1.0", "ra_text_edit 0.1.0", "rowan 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "smol_str 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/crates/ra_parser/Cargo.toml b/crates/ra_parser/Cargo.toml new file mode 100644 index 0000000000..b110e2bc6d --- /dev/null +++ b/crates/ra_parser/Cargo.toml @@ -0,0 +1,9 @@ +[package] +edition = "2018" +name = "ra_parser" +version = "0.1.0" +authors = ["rust-analyzer developers"] +publish = false + +[dependencies] +drop_bomb = "0.1.4" diff --git a/crates/ra_syntax/src/parsing/event.rs b/crates/ra_parser/src/event.rs similarity index 97% rename from crates/ra_syntax/src/parsing/event.rs rename to crates/ra_parser/src/event.rs index d6cbdffe08..d6e8454d43 100644 --- a/crates/ra_syntax/src/parsing/event.rs +++ b/crates/ra_parser/src/event.rs @@ -10,8 +10,8 @@ use std::mem; use crate::{ + ParseError, TreeSink, SyntaxKind::{self, *}, - parsing::{ParseError, TreeSink}, }; /// `Parser` produces a flat list of `Event`s. @@ -84,7 +84,7 @@ impl Event { } /// Generate the syntax tree with the control of events. -pub(super) fn process(sink: &mut impl TreeSink, mut events: Vec) { +pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec) { let mut forward_parents = Vec::new(); for i in 0..events.len() { diff --git a/crates/ra_syntax/src/parsing/grammar.rs b/crates/ra_parser/src/grammar.rs similarity index 97% rename from crates/ra_syntax/src/parsing/grammar.rs rename to crates/ra_parser/src/grammar.rs index 800d5a4a29..15aab6c6fc 100644 --- a/crates/ra_syntax/src/parsing/grammar.rs +++ b/crates/ra_parser/src/grammar.rs @@ -38,20 +38,18 @@ mod types; use crate::{ SyntaxKind::{self, *}, - parsing::{ - token_set::TokenSet, - parser::{CompletedMarker, Marker, Parser} - }, + TokenSet, + parser::{CompletedMarker, Marker, Parser}, }; -pub(super) fn root(p: &mut Parser) { +pub(crate) fn root(p: &mut Parser) { let m = p.start(); p.eat(SHEBANG); items::mod_contents(p, false); m.complete(p, SOURCE_FILE); } -pub(super) fn reparser( +pub(crate) fn reparser( node: SyntaxKind, first_child: Option, parent: Option, diff --git a/crates/ra_syntax/src/parsing/grammar/attributes.rs b/crates/ra_parser/src/grammar/attributes.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/attributes.rs rename to crates/ra_parser/src/grammar/attributes.rs diff --git a/crates/ra_syntax/src/parsing/grammar/expressions.rs b/crates/ra_parser/src/grammar/expressions.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/expressions.rs rename to crates/ra_parser/src/grammar/expressions.rs diff --git a/crates/ra_syntax/src/parsing/grammar/expressions/atom.rs b/crates/ra_parser/src/grammar/expressions/atom.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/expressions/atom.rs rename to crates/ra_parser/src/grammar/expressions/atom.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items.rs b/crates/ra_parser/src/grammar/items.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items.rs rename to crates/ra_parser/src/grammar/items.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/consts.rs b/crates/ra_parser/src/grammar/items/consts.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/consts.rs rename to crates/ra_parser/src/grammar/items/consts.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/nominal.rs b/crates/ra_parser/src/grammar/items/nominal.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/nominal.rs rename to crates/ra_parser/src/grammar/items/nominal.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/traits.rs b/crates/ra_parser/src/grammar/items/traits.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/traits.rs rename to crates/ra_parser/src/grammar/items/traits.rs diff --git a/crates/ra_syntax/src/parsing/grammar/items/use_item.rs b/crates/ra_parser/src/grammar/items/use_item.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/items/use_item.rs rename to crates/ra_parser/src/grammar/items/use_item.rs diff --git a/crates/ra_syntax/src/parsing/grammar/params.rs b/crates/ra_parser/src/grammar/params.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/params.rs rename to crates/ra_parser/src/grammar/params.rs diff --git a/crates/ra_syntax/src/parsing/grammar/paths.rs b/crates/ra_parser/src/grammar/paths.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/paths.rs rename to crates/ra_parser/src/grammar/paths.rs diff --git a/crates/ra_syntax/src/parsing/grammar/patterns.rs b/crates/ra_parser/src/grammar/patterns.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/patterns.rs rename to crates/ra_parser/src/grammar/patterns.rs diff --git a/crates/ra_syntax/src/parsing/grammar/type_args.rs b/crates/ra_parser/src/grammar/type_args.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/type_args.rs rename to crates/ra_parser/src/grammar/type_args.rs diff --git a/crates/ra_syntax/src/parsing/grammar/type_params.rs b/crates/ra_parser/src/grammar/type_params.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/type_params.rs rename to crates/ra_parser/src/grammar/type_params.rs diff --git a/crates/ra_syntax/src/parsing/grammar/types.rs b/crates/ra_parser/src/grammar/types.rs similarity index 100% rename from crates/ra_syntax/src/parsing/grammar/types.rs rename to crates/ra_parser/src/grammar/types.rs diff --git a/crates/ra_parser/src/lib.rs b/crates/ra_parser/src/lib.rs new file mode 100644 index 0000000000..7931b5189f --- /dev/null +++ b/crates/ra_parser/src/lib.rs @@ -0,0 +1,64 @@ +#[macro_use] +mod token_set; +mod syntax_kind; +mod event; +mod parser; +mod grammar; + +pub(crate) use token_set::TokenSet; + +pub use syntax_kind::SyntaxKind; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ParseError(pub String); + +/// `TreeSink` abstracts details of a particular syntax tree implementation. +pub trait TreeSink { + /// Adds new leaf to the current branch. + fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8); + + /// Start new branch and make it current. + fn start_branch(&mut self, kind: SyntaxKind, root: bool); + + /// Finish current branch and restore previous + /// branch as current. + fn finish_branch(&mut self, root: bool); + + fn error(&mut self, error: ParseError); +} + +/// `TokenSource` abstracts the source of the tokens parser operates one. +/// +/// Hopefully this will allow us to treat text and token trees in the same way! +pub trait TokenSource { + fn token_kind(&self, pos: usize) -> SyntaxKind; + fn is_token_joint_to_next(&self, pos: usize) -> bool; + fn is_keyword(&self, pos: usize, kw: &str) -> bool; +} + +pub fn parse(token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + let mut p = parser::Parser::new(token_source); + grammar::root(&mut p); + let events = p.finish(); + event::process(tree_sink, events); +} + +pub struct Reparser(fn(&mut parser::Parser)); + +impl Reparser { + pub fn for_node( + node: SyntaxKind, + first_child: Option, + parent: Option, + ) -> Option { + grammar::reparser(node, first_child, parent).map(Reparser) + } + + pub fn parse(self, token_source: &dyn TokenSource, tree_sink: &mut dyn TreeSink) { + let Reparser(r) = self; + let mut p = parser::Parser::new(token_source); + r(&mut p); + let events = p.finish(); + event::process(tree_sink, events); + } +} diff --git a/crates/ra_syntax/src/parsing/parser.rs b/crates/ra_parser/src/parser.rs similarity index 98% rename from crates/ra_syntax/src/parsing/parser.rs rename to crates/ra_parser/src/parser.rs index 923b0f2b20..a18458148a 100644 --- a/crates/ra_syntax/src/parsing/parser.rs +++ b/crates/ra_parser/src/parser.rs @@ -4,11 +4,8 @@ use drop_bomb::DropBomb; use crate::{ SyntaxKind::{self, ERROR, EOF, TOMBSTONE}, - parsing::{ - TokenSource, ParseError, - token_set::TokenSet, - event::Event, - }, + TokenSource, ParseError, TokenSet, + event::Event, }; /// `Parser` struct provides the low-level API for diff --git a/crates/ra_syntax/src/syntax_kinds.rs b/crates/ra_parser/src/syntax_kind.rs similarity index 85% rename from crates/ra_syntax/src/syntax_kinds.rs rename to crates/ra_parser/src/syntax_kind.rs index c1118c5ab4..a2353317f8 100644 --- a/crates/ra_syntax/src/syntax_kinds.rs +++ b/crates/ra_parser/src/syntax_kind.rs @@ -2,8 +2,6 @@ mod generated; use std::fmt; -use crate::SyntaxKind::*; - pub use self::generated::SyntaxKind; impl fmt::Debug for SyntaxKind { @@ -20,7 +18,7 @@ pub(crate) struct SyntaxInfo { impl SyntaxKind { pub fn is_trivia(self) -> bool { match self { - WHITESPACE | COMMENT => true, + SyntaxKind::WHITESPACE | SyntaxKind::COMMENT => true, _ => false, } } diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs b/crates/ra_parser/src/syntax_kind/generated.rs similarity index 99% rename from crates/ra_syntax/src/syntax_kinds/generated.rs rename to crates/ra_parser/src/syntax_kind/generated.rs index 266b95bbb1..1d8f988aed 100644 --- a/crates/ra_syntax/src/syntax_kinds/generated.rs +++ b/crates/ra_parser/src/syntax_kind/generated.rs @@ -568,7 +568,7 @@ impl SyntaxKind { EOF => &SyntaxInfo { name: "EOF" }, } } - pub(crate) fn from_keyword(ident: &str) -> Option { + pub fn from_keyword(ident: &str) -> Option { let kw = match ident { "use" => USE_KW, "fn" => FN_KW, @@ -610,7 +610,7 @@ impl SyntaxKind { Some(kw) } - pub(crate) fn from_char(c: char) -> Option { + pub fn from_char(c: char) -> Option { let tok = match c { ';' => SEMI, ',' => COMMA, diff --git a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera b/crates/ra_parser/src/syntax_kind/generated.rs.tera similarity index 95% rename from crates/ra_syntax/src/syntax_kinds/generated.rs.tera rename to crates/ra_parser/src/syntax_kind/generated.rs.tera index 837437136d..f241a21a0e 100644 --- a/crates/ra_syntax/src/syntax_kinds/generated.rs.tera +++ b/crates/ra_parser/src/syntax_kind/generated.rs.tera @@ -74,7 +74,7 @@ impl SyntaxKind { EOF => &SyntaxInfo { name: "EOF" }, } } - pub(crate) fn from_keyword(ident: &str) -> Option { + pub fn from_keyword(ident: &str) -> Option { let kw = match ident { {%- for kw in keywords %} "{{kw}}" => {{kw | upper}}_KW, @@ -84,7 +84,7 @@ impl SyntaxKind { Some(kw) } - pub(crate) fn from_char(c: char) -> Option { + pub fn from_char(c: char) -> Option { let tok = match c { {%- for t in single_byte_tokens %} '{{t.0}}' => {{t.1}}, diff --git a/crates/ra_syntax/src/parsing/token_set.rs b/crates/ra_parser/src/token_set.rs similarity index 100% rename from crates/ra_syntax/src/parsing/token_set.rs rename to crates/ra_parser/src/token_set.rs diff --git a/crates/ra_syntax/Cargo.toml b/crates/ra_syntax/Cargo.toml index 7ce26b7c4d..7e70dad3ff 100644 --- a/crates/ra_syntax/Cargo.toml +++ b/crates/ra_syntax/Cargo.toml @@ -21,6 +21,7 @@ text_unit = { version = "0.1.6", features = ["serde"] } smol_str = { version = "0.1.9", features = ["serde"] } ra_text_edit = { path = "../ra_text_edit" } +ra_parser = { path = "../ra_parser" } [dev-dependencies] test_utils = { path = "../test_utils" } diff --git a/crates/ra_syntax/src/lib.rs b/crates/ra_syntax/src/lib.rs index b12282b394..c788bddec8 100644 --- a/crates/ra_syntax/src/lib.rs +++ b/crates/ra_syntax/src/lib.rs @@ -16,7 +16,6 @@ #![allow(missing_docs)] //#![warn(unreachable_pub)] // rust-lang/rust#47816 -mod syntax_kinds; mod syntax_node; mod syntax_text; mod syntax_error; @@ -31,9 +30,9 @@ pub mod ast; pub mod utils; pub use rowan::{SmolStr, TextRange, TextUnit}; +pub use ra_parser::SyntaxKind; pub use crate::{ ast::AstNode, - syntax_kinds::SyntaxKind, syntax_error::{SyntaxError, SyntaxErrorKind, Location}, syntax_text::SyntaxText, syntax_node::{Direction, SyntaxNode, WalkEvent, TreeArc}, diff --git a/crates/ra_syntax/src/parsing.rs b/crates/ra_syntax/src/parsing.rs index 7e1b32035b..0a11600e19 100644 --- a/crates/ra_syntax/src/parsing.rs +++ b/crates/ra_syntax/src/parsing.rs @@ -1,50 +1,28 @@ -#[macro_use] -mod token_set; mod builder; mod lexer; -mod event; mod input; -mod parser; -mod grammar; mod reparsing; +use ra_parser::{parse, ParseError}; + use crate::{ SyntaxKind, SyntaxError, parsing::{ builder::TreeBuilder, input::ParserInput, - event::process, - parser::Parser, }, syntax_node::GreenNode, }; pub use self::lexer::{tokenize, Token}; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ParseError(pub String); - pub(crate) use self::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec) { let tokens = tokenize(&text); - let tree_sink = TreeBuilder::new(text, &tokens); - parse_with(tree_sink, text, &tokens, grammar::root) -} - -fn parse_with( - mut tree_sink: S, - text: &str, - tokens: &[Token], - f: fn(&mut Parser), -) -> S::Tree { - let events = { - let input = ParserInput::new(text, &tokens); - let mut p = Parser::new(&input); - f(&mut p); - p.finish() - }; - process(&mut tree_sink, events); + let token_source = ParserInput::new(text, &tokens); + let mut tree_sink = TreeBuilder::new(text, &tokens); + parse(&token_source, &mut tree_sink); tree_sink.finish() } diff --git a/crates/ra_syntax/src/parsing/builder.rs b/crates/ra_syntax/src/parsing/builder.rs index 1041c6a7be..0775b09006 100644 --- a/crates/ra_syntax/src/parsing/builder.rs +++ b/crates/ra_syntax/src/parsing/builder.rs @@ -1,7 +1,9 @@ +use ra_parser::{TreeSink, ParseError}; + use crate::{ SmolStr, SyntaxError, SyntaxErrorKind, TextUnit, TextRange, SyntaxKind::{self, *}, - parsing::{TreeSink, ParseError, Token}, + parsing::Token, syntax_node::{GreenNode, RaTypes}, }; @@ -17,8 +19,6 @@ pub(crate) struct TreeBuilder<'a> { } impl<'a> TreeSink for TreeBuilder<'a> { - type Tree = (GreenNode, Vec); - fn leaf(&mut self, kind: SyntaxKind, n_tokens: u8) { self.eat_trivias(); let n_tokens = n_tokens as usize; @@ -65,10 +65,6 @@ impl<'a> TreeSink for TreeBuilder<'a> { let error = SyntaxError::new(SyntaxErrorKind::ParseError(error), self.text_pos); self.errors.push(error) } - - fn finish(self) -> (GreenNode, Vec) { - (self.inner.finish(), self.errors) - } } impl<'a> TreeBuilder<'a> { @@ -82,6 +78,11 @@ impl<'a> TreeBuilder<'a> { inner: GreenNodeBuilder::new(), } } + + pub(super) fn finish(self) -> (GreenNode, Vec) { + (self.inner.finish(), self.errors) + } + fn eat_trivias(&mut self) { while let Some(&token) = self.tokens.get(self.token_pos) { if !token.kind.is_trivia() { diff --git a/crates/ra_syntax/src/parsing/input.rs b/crates/ra_syntax/src/parsing/input.rs index 96c03bb118..58be795bc3 100644 --- a/crates/ra_syntax/src/parsing/input.rs +++ b/crates/ra_syntax/src/parsing/input.rs @@ -1,9 +1,8 @@ +use ra_parser::TokenSource; + use crate::{ SyntaxKind, SyntaxKind::EOF, TextRange, TextUnit, - parsing::{ - TokenSource, - lexer::Token, - }, + parsing::lexer::Token, }; impl<'t> TokenSource for ParserInput<'t> { diff --git a/crates/ra_syntax/src/parsing/reparsing.rs b/crates/ra_syntax/src/parsing/reparsing.rs index 2c860b3df0..ffcb512adf 100644 --- a/crates/ra_syntax/src/parsing/reparsing.rs +++ b/crates/ra_syntax/src/parsing/reparsing.rs @@ -1,18 +1,18 @@ +use ra_text_edit::AtomTextEdit; +use ra_parser::Reparser; + use crate::{ SyntaxKind::*, TextRange, TextUnit, algo, syntax_node::{GreenNode, SyntaxNode}, syntax_error::SyntaxError, parsing::{ - grammar, parse_with, + input::ParserInput, builder::TreeBuilder, - parser::Parser, lexer::{tokenize, Token}, } }; -use ra_text_edit::AtomTextEdit; - pub(crate) fn incremental_reparse( node: &SyntaxNode, edit: &AtomTextEdit, @@ -61,8 +61,10 @@ fn reparse_block<'node>( if !is_balanced(&tokens) { return None; } - let tree_sink = TreeBuilder::new(&text, &tokens); - let (green, new_errors) = parse_with(tree_sink, &text, &tokens, reparser); + let token_source = ParserInput::new(&text, &tokens); + let mut tree_sink = TreeBuilder::new(&text, &tokens); + reparser.parse(&token_source, &mut tree_sink); + let (green, new_errors) = tree_sink.finish(); Some((node, green, new_errors)) } @@ -78,15 +80,12 @@ fn is_contextual_kw(text: &str) -> bool { } } -fn find_reparsable_node( - node: &SyntaxNode, - range: TextRange, -) -> Option<(&SyntaxNode, fn(&mut Parser))> { +fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(&SyntaxNode, Reparser)> { let node = algo::find_covering_node(node, range); node.ancestors().find_map(|node| { let first_child = node.first_child().map(|it| it.kind()); let parent = node.parent().map(|it| it.kind()); - grammar::reparser(node.kind(), first_child, parent).map(|r| (node, r)) + Reparser::for_node(node.kind(), first_child, parent).map(|r| (node, r)) }) } diff --git a/crates/ra_syntax/src/syntax_error.rs b/crates/ra_syntax/src/syntax_error.rs index 1a00fcc27a..bdd4317422 100644 --- a/crates/ra_syntax/src/syntax_error.rs +++ b/crates/ra_syntax/src/syntax_error.rs @@ -1,6 +1,8 @@ use std::fmt; -use crate::{TextRange, TextUnit, parsing::ParseError}; +use ra_parser::ParseError; + +use crate::{TextRange, TextUnit}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct SyntaxError { diff --git a/crates/tools/src/lib.rs b/crates/tools/src/lib.rs index 0a10d2737e..3c23ed76e0 100644 --- a/crates/tools/src/lib.rs +++ b/crates/tools/src/lib.rs @@ -14,11 +14,11 @@ pub use teraron::{Mode, Overwrite, Verify}; pub type Result = std::result::Result; pub const GRAMMAR: &str = "crates/ra_syntax/src/grammar.ron"; -const GRAMMAR_DIR: &str = "crates/ra_syntax/src/parsing/grammar"; +const GRAMMAR_DIR: &str = "crates/ra_parser/src/grammar"; const OK_INLINE_TESTS_DIR: &str = "crates/ra_syntax/tests/data/parser/inline/ok"; const ERR_INLINE_TESTS_DIR: &str = "crates/ra_syntax/tests/data/parser/inline/err"; -pub const SYNTAX_KINDS: &str = "crates/ra_syntax/src/syntax_kinds/generated.rs.tera"; +pub const SYNTAX_KINDS: &str = "crates/ra_parser/src/syntax_kind/generated.rs.tera"; pub const AST: &str = "crates/ra_syntax/src/ast/generated.rs.tera"; const TOOLCHAIN: &str = "stable";