merge parse_impl and parser_api

crates/ra_syntax/src/parsing/parser_api.rs

@@ -1,10 +1,14 @@
+use std::cell::Cell;
+
 use drop_bomb::DropBomb;
 
 use crate::{
-    SyntaxKind::{self, ERROR},
+    syntax_error::ParseError,
+    SyntaxKind::{self, ERROR, EOF, TOMBSTONE},
     parsing::{
+        TokenSource, TokenPos,
         token_set::TokenSet,
-        parser_impl::ParserImpl,
+        parser_impl::event::Event,
     },
 };
 
@@ -17,9 +21,22 @@ use crate::{
 /// tree, but rather a flat stream of events of the form
 /// "start expression, consume number literal,
 /// finish expression". See `Event` docs for more.
-pub(crate) struct Parser<'t>(pub(super) ParserImpl<'t>);
+pub(crate) struct Parser<'t> {
+    token_source: &'t dyn TokenSource,
+    pos: TokenPos,
+    events: Vec<Event>,
+    steps: Cell<u32>,
+}
 
 impl<'t> Parser<'t> {
+    pub(super) fn new(token_source: &'t dyn TokenSource) -> Parser<'t> {
+        Parser { token_source, pos: TokenPos::default(), events: Vec::new(), steps: Cell::new(0) }
+    }
+
+    pub(crate) fn finish(self) -> Vec<Event> {
+        self.events
+    }
+
     /// Returns the kind of the current token.
     /// If parser has already reached the end of input,
     /// the special `EOF` kind is returned.
@@ -32,7 +49,13 @@ impl<'t> Parser<'t> {
     ///
     /// Useful for parsing things like `>>`.
     pub(crate) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
-        self.0.current2()
+        let c1 = self.token_source.token_kind(self.pos);
+        let c2 = self.token_source.token_kind(self.pos + 1);
+        if self.token_source.is_token_joint_to_next(self.pos) {
+            Some((c1, c2))
+        } else {
+            None
+        }
     }
 
     /// Returns the kinds of the current three tokens, if they are not separated
@@ -40,13 +63,25 @@ impl<'t> Parser<'t> {
     ///
     /// Useful for parsing things like `=>>`.
     pub(crate) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
-        self.0.current3()
+        let c1 = self.token_source.token_kind(self.pos);
+        let c2 = self.token_source.token_kind(self.pos + 1);
+        let c3 = self.token_source.token_kind(self.pos + 2);
+        if self.token_source.is_token_joint_to_next(self.pos)
+            && self.token_source.is_token_joint_to_next(self.pos + 1)
+        {
+            Some((c1, c2, c3))
+        } else {
+            None
+        }
     }
 
     /// Lookahead operation: returns the kind of the next nth
     /// token.
     pub(crate) fn nth(&self, n: u32) -> SyntaxKind {
-        self.0.nth(n)
+        let steps = self.steps.get();
+        assert!(steps <= 10_000_000, "the parser seems stuck");
+        self.steps.set(steps + 1);
+        self.token_source.token_kind(self.pos + n)
     }
 
     /// Checks if the current token is `kind`.
@@ -60,20 +95,26 @@ impl<'t> Parser<'t> {
     }
 
     /// Checks if the current token is contextual keyword with text `t`.
-    pub(crate) fn at_contextual_kw(&self, t: &str) -> bool {
+    pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
-        self.0.at_kw(t)
+        self.token_source.is_keyword(self.pos, kw)
     }
 
     /// Starts a new node in the syntax tree. All nodes and tokens
     /// consumed between the `start` and the corresponding `Marker::complete`
     /// belong to the same node.
     pub(crate) fn start(&mut self) -> Marker {
-        Marker::new(self.0.start())
+        let pos = self.events.len() as u32;
+        self.push_event(Event::tombstone());
+        Marker::new(pos)
     }
 
     /// Advances the parser by one token unconditionally.
     pub(crate) fn bump(&mut self) {
-        self.0.bump();
+        let kind = self.nth(0);
+        if kind == EOF {
+            return;
+        }
+        self.do_bump(kind, 1);
     }
 
     /// Advances the parser by one token, remapping its kind.
@@ -83,14 +124,18 @@ impl<'t> Parser<'t> {
     /// `union` keyword, and keyword is what ends up in the
     /// final tree.
     pub(crate) fn bump_remap(&mut self, kind: SyntaxKind) {
-        self.0.bump_remap(kind);
+        if self.nth(0) == EOF {
+            // TODO: panic!?
+            return;
+        }
+        self.do_bump(kind, 1);
     }
 
     /// Advances the parser by `n` tokens, remapping its kind.
     /// This is useful to create compound tokens from parts. For
     /// example, an `<<` token is two consecutive remapped `<` tokens
     pub(crate) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) {
-        self.0.bump_compound(kind, n);
+        self.do_bump(kind, n);
     }
 
     /// Emit error with the `message`
@@ -98,7 +143,8 @@ impl<'t> Parser<'t> {
     /// structured errors with spans and notes, like rustc
     /// does.
     pub(crate) fn error<T: Into<String>>(&mut self, message: T) {
-        self.0.error(message.into())
+        let msg = ParseError(message.into());
+        self.push_event(Event::Error { msg })
     }
 
     /// Consume the next token if `kind` matches.
@@ -136,6 +182,15 @@ impl<'t> Parser<'t> {
             m.complete(self, ERROR);
         };
     }
+
+    fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
+        self.pos += u32::from(n_raw_tokens);
+        self.push_event(Event::Token { kind, n_raw_tokens });
+    }
+
+    fn push_event(&mut self, event: Event) {
+        self.events.push(event)
+    }
 }
 
 /// See `Parser::start`.
@@ -154,7 +209,14 @@ impl Marker {
     /// operation like `.precede()` to deal with forward_parent.
     pub(crate) fn complete(mut self, p: &mut Parser, kind: SyntaxKind) -> CompletedMarker {
         self.bomb.defuse();
-        p.0.complete(self.pos, kind);
+        let idx = self.pos as usize;
+        match p.events[idx] {
+            Event::Start { kind: ref mut slot, .. } => {
+                *slot = kind;
+            }
+            _ => unreachable!(),
+        }
+        p.push_event(Event::Finish);
         CompletedMarker::new(self.pos, kind)
     }
 
@@ -162,7 +224,13 @@ impl Marker {
     /// are attached to its parent instead.
     pub(crate) fn abandon(mut self, p: &mut Parser) {
         self.bomb.defuse();
-        p.0.abandon(self.pos);
+        let idx = self.pos as usize;
+        if idx == p.events.len() - 1 {
+            match p.events.pop() {
+                Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (),
+                _ => unreachable!(),
+            }
+        }
     }
 }
 
@@ -186,7 +254,15 @@ impl CompletedMarker {
     /// then mark `NEWSTART` as `START`'s parent with saving its relative
     /// distance to `NEWSTART` into forward_parent(=2 in this case);
     pub(crate) fn precede(self, p: &mut Parser) -> Marker {
-        Marker::new(p.0.precede(self.0))
+        let new_pos = p.start();
+        let idx = self.0 as usize;
+        match p.events[idx] {
+            Event::Start { ref mut forward_parent, .. } => {
+                *forward_parent = Some(new_pos.pos - self.0);
+            }
+            _ => unreachable!(),
+        }
+        new_pos
     }
 
     pub(crate) fn kind(&self) -> SyntaxKind {

crates/ra_syntax/src/parsing/parser_impl.rs

@@ -1,20 +1,13 @@
-mod event;
+pub(super) mod event;
-pub(crate) mod input;
+pub(super) mod input;
 
-use std::cell::Cell;
+use crate::parsing::{
-
+    TreeSink, TokenSource,
-use crate::{
-    syntax_error::ParseError,
-    parsing::{
-        TreeSink, TokenSource, TokenPos,
     lexer::Token,
     parser_api::Parser,
-        parser_impl::event::{Event, EventProcessor},
+    parser_impl::event::EventProcessor,
-    },
 };
 
-use crate::SyntaxKind::{self, EOF, TOMBSTONE};
-
 /// Parse a sequence of tokens into the representative node tree
 pub(super) fn parse_with<S: TreeSink>(
     sink: S,
@@ -24,147 +17,9 @@ pub(super) fn parse_with<S: TreeSink>(
 ) -> S::Tree {
     let mut events = {
         let input = input::ParserInput::new(text, tokens);
-        let parser_impl = ParserImpl::new(&input);
+        let mut parser_api = Parser::new(&input);
-        let mut parser_api = Parser(parser_impl);
         parser(&mut parser_api);
-        parser_api.0.into_events()
+        parser_api.finish()
     };
     EventProcessor::new(sink, text, tokens, &mut events).process().finish()
 }
-
-/// Implementation details of `Parser`, extracted
-/// to a separate struct in order not to pollute
-/// the public API of the `Parser`.
-pub(super) struct ParserImpl<'a> {
-    token_source: &'a dyn TokenSource,
-    pos: TokenPos,
-    events: Vec<Event>,
-    steps: Cell<u32>,
-}
-
-impl<'a> ParserImpl<'a> {
-    fn new(token_source: &'a dyn TokenSource) -> ParserImpl<'a> {
-        ParserImpl {
-            token_source,
-            pos: TokenPos::default(),
-            events: Vec::new(),
-            steps: Cell::new(0),
-        }
-    }
-
-    fn into_events(self) -> Vec<Event> {
-        assert_eq!(self.nth(0), EOF);
-        self.events
-    }
-
-    pub(super) fn current2(&self) -> Option<(SyntaxKind, SyntaxKind)> {
-        let c1 = self.token_source.token_kind(self.pos);
-        let c2 = self.token_source.token_kind(self.pos + 1);
-        if self.token_source.is_token_joint_to_next(self.pos) {
-            Some((c1, c2))
-        } else {
-            None
-        }
-    }
-
-    pub(super) fn current3(&self) -> Option<(SyntaxKind, SyntaxKind, SyntaxKind)> {
-        let c1 = self.token_source.token_kind(self.pos);
-        let c2 = self.token_source.token_kind(self.pos + 1);
-        let c3 = self.token_source.token_kind(self.pos + 2);
-        if self.token_source.is_token_joint_to_next(self.pos)
-            && self.token_source.is_token_joint_to_next(self.pos + 1)
-        {
-            Some((c1, c2, c3))
-        } else {
-            None
-        }
-    }
-
-    /// Get the syntax kind of the nth token.
-    pub(super) fn nth(&self, n: u32) -> SyntaxKind {
-        let steps = self.steps.get();
-        assert!(steps <= 10_000_000, "the parser seems stuck");
-        self.steps.set(steps + 1);
-        self.token_source.token_kind(self.pos + n)
-    }
-
-    pub(super) fn at_kw(&self, kw: &str) -> bool {
-        self.token_source.is_keyword(self.pos, kw)
-    }
-
-    /// Start parsing right behind the last event.
-    pub(super) fn start(&mut self) -> u32 {
-        let pos = self.events.len() as u32;
-        self.push_event(Event::tombstone());
-        pos
-    }
-
-    /// Advances the parser by one token unconditionally.
-    pub(super) fn bump(&mut self) {
-        let kind = self.nth(0);
-        if kind == EOF {
-            return;
-        }
-        self.do_bump(kind, 1);
-    }
-
-    pub(super) fn bump_remap(&mut self, kind: SyntaxKind) {
-        if self.nth(0) == EOF {
-            // TODO: panic!?
-            return;
-        }
-        self.do_bump(kind, 1);
-    }
-
-    pub(super) fn bump_compound(&mut self, kind: SyntaxKind, n: u8) {
-        self.do_bump(kind, n);
-    }
-
-    fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
-        self.pos += u32::from(n_raw_tokens);
-        self.push_event(Event::Token { kind, n_raw_tokens });
-    }
-
-    /// Append one Error event to the back of events.
-    pub(super) fn error(&mut self, msg: String) {
-        self.push_event(Event::Error { msg: ParseError(msg) })
-    }
-
-    /// Complete an event with appending a `Finish` event.
-    pub(super) fn complete(&mut self, pos: u32, kind: SyntaxKind) {
-        match self.events[pos as usize] {
-            Event::Start { kind: ref mut slot, .. } => {
-                *slot = kind;
-            }
-            _ => unreachable!(),
-        }
-        self.push_event(Event::Finish);
-    }
-
-    /// Ignore the dummy `Start` event.
-    pub(super) fn abandon(&mut self, pos: u32) {
-        let idx = pos as usize;
-        if idx == self.events.len() - 1 {
-            match self.events.pop() {
-                Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (),
-                _ => unreachable!(),
-            }
-        }
-    }
-
-    /// Save the relative distance of a completed event to its forward_parent.
-    pub(super) fn precede(&mut self, pos: u32) -> u32 {
-        let new_pos = self.start();
-        match self.events[pos as usize] {
-            Event::Start { ref mut forward_parent, .. } => {
-                *forward_parent = Some(new_pos - pos);
-            }
-            _ => unreachable!(),
-        }
-        new_pos
-    }
-
-    fn push_event(&mut self, event: Event) {
-        self.events.push(event)
-    }
-}