language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-30 22:01:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

internal: replace TreeSink with a data structure

Browse source 
The general theme of this is to make parser a better independent
library.

The specific thing we do here is replacing callback based TreeSink with
a data structure. That is, rather than calling user-provided tree
construction methods, the parser now spits out a very bare-bones tree,
effectively a log of a DFS traversal.

This makes the parser usable without any *specifc* tree sink, and allows
us to, eg, move tests into this crate.

Now, it's also true that this is a distinction without a difference, as
the old and the new interface are equivalent in expressiveness. Still,
this new thing seems somewhat simpler. But yeah, I admit I don't have a
suuper strong motivation here, just a hunch that this is better.
This commit is contained in:
Aleksey Kladov 2021-12-19 17:36:23 +03:00
parent 2f63558dc5
commit d0d05075ed
10 changed files with 172 additions and 110 deletions
Download patch file Download diff file Expand all files Collapse all files

49
crates/mbe/src/tt_iter.rs
View file

@ -3,9 +3,8 @@
use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};
use parser::TreeSink;
use syntax::SyntaxKind;
use tt::buffer::{Cursor, TokenBuffer};
use tt::buffer::TokenBuffer;
macro_rules! err {
() => {
@ -94,34 +93,28 @@ impl<'a> TtIter<'a> {
&mut self,
entry_point: ParserEntryPoint,
) -> ExpandResult<Option<tt::TokenTree>> {
struct OffsetTokenSink<'a> {
cursor: Cursor<'a>,
error: bool,
}
impl<'a> TreeSink for OffsetTokenSink<'a> {
fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {
if kind == SyntaxKind::LIFETIME_IDENT {
n_tokens = 2;
}
for _ in 0..n_tokens {
self.cursor = self.cursor.bump_subtree();
}
}
fn start_node(&mut self, _kind: SyntaxKind) {}
fn finish_node(&mut self) {}
fn error(&mut self, _error: parser::ParseError) {
self.error = true;
}
}
let buffer = TokenBuffer::from_tokens(self.inner.as_slice());
let parser_tokens = to_parser_tokens(&buffer);
let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };
let tree_traversal = parser::parse(&parser_tokens, entry_point);
parser::parse(&parser_tokens, &mut sink, entry_point);
let mut cursor = buffer.begin();
let mut error = false;
for step in tree_traversal.iter() {
match step {
parser::TraversalStep::Token { kind, mut n_raw_tokens } => {
if kind == SyntaxKind::LIFETIME_IDENT {
n_raw_tokens = 2;
}
for _ in 0..n_raw_tokens {
cursor = cursor.bump_subtree();
}
}
parser::TraversalStep::EnterNode { .. } | parser::TraversalStep::LeaveNode => (),
parser::TraversalStep::Error { .. } => error = true,
}
}
let mut err = if !sink.cursor.is_root() || sink.error {
let mut err = if !cursor.is_root() || error {
Some(err!("expected {:?}", entry_point))
} else {
None
@ -130,8 +123,8 @@ impl<'a> TtIter<'a> {
let mut curr = buffer.begin();
let mut res = vec![];
if sink.cursor.is_root() {
while curr != sink.cursor {
if cursor.is_root() {
while curr != cursor {
if let Some(token) = curr.token_tree() {
res.push(token);
}