language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-26 20:09:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Change TokenSource to iteration based

Browse source
This commit is contained in:
Edwin Cheng 2019-05-25 20:31:53 +08:00
parent ef00b5af1c
commit fcb1eef323
8 changed files with 174 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/ra_mbe/src/subtree_parser.rs
View file

@ -68,13 +68,13 @@ impl<'a> Parser<'a> {
fn parse<F>(self, f: F) -> Option<tt::TokenTree>
where
F: FnOnce(&dyn TokenSource, &mut dyn TreeSink),
F: FnOnce(&mut dyn TokenSource, &mut dyn TreeSink),
{
let buffer = TokenBuffer::new(&self.subtree.token_trees[*self.cur_pos..]);
let mut src = SubtreeTokenSource::new(&buffer);
let mut sink = OffsetTokenSink { token_pos: 0, error: false };
f(&src, &mut sink);
f(&mut src, &mut sink);
let r = self.finish(sink.token_pos, &mut src);
if sink.error {

61
crates/ra_mbe/src/subtree_source.rs
View file

@ -1,6 +1,7 @@
use ra_parser::{TokenSource};
use ra_parser::{TokenSource, Token};
use ra_syntax::{classify_literal, SmolStr, SyntaxKind, SyntaxKind::*, T};
use std::cell::{RefCell, Cell};
use std::sync::Arc;
use tt::buffer::{TokenBuffer, Cursor};
pub(crate) trait Querier {
@ -65,7 +66,7 @@ impl<'a> SubtreeWalk<'a> {
return cached[pos].clone();
}
fn collect_token_trees(&mut self, n: usize) -> Vec<tt::TokenTree> {
fn collect_token_trees(&self, n: usize) -> Vec<tt::TokenTree> {
let mut res = vec![];
let mut pos = 0;
@ -117,43 +118,59 @@ impl<'a> Querier for SubtreeWalk<'a> {
}
pub(crate) struct SubtreeTokenSource<'a> {
walker: SubtreeWalk<'a>,
walker: Arc<SubtreeWalk<'a>>,
curr: (Token, usize),
}
impl<'a> SubtreeTokenSource<'a> {
pub fn new(buffer: &'a TokenBuffer) -> SubtreeTokenSource<'a> {
SubtreeTokenSource { walker: SubtreeWalk::new(buffer.begin()) }
let mut res = SubtreeTokenSource {
walker: Arc::new(SubtreeWalk::new(buffer.begin())),
curr: (Token { kind: EOF, is_jointed_to_next: false }, 0),
};
res.curr = (res.mk_token(0), 0);
res
}
pub fn querier<'b>(&'a self) -> &'b SubtreeWalk<'a>
where
'a: 'b,
{
&self.walker
pub fn querier(&self) -> Arc<SubtreeWalk<'a>> {
self.walker.clone()
}
pub(crate) fn bump_n(&mut self, parsed_tokens: usize) -> Vec<tt::TokenTree> {
let res = self.walker.collect_token_trees(parsed_tokens);
res
}
fn mk_token(&self, pos: usize) -> Token {
match self.walker.get(pos) {
Some(tt) => Token { kind: tt.kind, is_jointed_to_next: tt.is_joint_to_next },
None => Token { kind: EOF, is_jointed_to_next: false },
}
}
}
impl<'a> TokenSource for SubtreeTokenSource<'a> {
fn token_kind(&self, pos: usize) -> SyntaxKind {
if let Some(tok) = self.walker.get(pos) {
tok.kind
} else {
SyntaxKind::EOF
}
fn current(&self) -> Token {
self.curr.0
}
fn is_token_joint_to_next(&self, pos: usize) -> bool {
match self.walker.get(pos) {
Some(t) => t.is_joint_to_next,
_ => false,
}
/// Lookahead n token
fn lookahead_nth(&self, n: usize) -> Token {
self.mk_token(self.curr.1 + n)
}
fn is_keyword(&self, pos: usize, kw: &str) -> bool {
match self.walker.get(pos) {
/// bump cursor to next token
fn bump(&mut self) {
if self.current().kind == EOF {
return;
}
self.curr = (self.mk_token(self.curr.1 + 1), self.curr.1 + 1)
}
/// Is the current token a specified keyword?
fn is_keyword(&self, kw: &str) -> bool {
match self.walker.get(self.curr.1) {
Some(t) => t.text == *kw,
_ => false,
}

42
crates/ra_mbe/src/syntax_bridge.rs
View file

@ -48,9 +48,10 @@ pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> Option<(tt::Subtree, Toke
/// Parses the token tree (result of macro expansion) to an expression
pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, ExpandError> {
let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
let token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(token_source.querier());
ra_parser::parse_expr(&token_source, &mut tree_sink);
let mut token_source = SubtreeTokenSource::new(&buffer);
let querier = token_source.querier();
let mut tree_sink = TtTreeSink::new(querier.as_ref());
ra_parser::parse_expr(&mut token_source, &mut tree_sink);
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
@ -64,9 +65,10 @@ pub fn token_tree_to_expr(tt: &tt::Subtree) -> Result<TreeArc<ast::Expr>, Expand
/// Parses the token tree (result of macro expansion) to a Pattern
pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandError> {
let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
let token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(token_source.querier());
ra_parser::parse_pat(&token_source, &mut tree_sink);
let mut token_source = SubtreeTokenSource::new(&buffer);
let querier = token_source.querier();
let mut tree_sink = TtTreeSink::new(querier.as_ref());
ra_parser::parse_pat(&mut token_source, &mut tree_sink);
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
@ -78,9 +80,10 @@ pub fn token_tree_to_pat(tt: &tt::Subtree) -> Result<TreeArc<ast::Pat>, ExpandEr
/// Parses the token tree (result of macro expansion) to a Type
pub fn token_tree_to_ty(tt: &tt::Subtree) -> Result<TreeArc<ast::TypeRef>, ExpandError> {
let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
let token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(token_source.querier());
ra_parser::parse_ty(&token_source, &mut tree_sink);
let mut token_source = SubtreeTokenSource::new(&buffer);
let querier = token_source.querier();
let mut tree_sink = TtTreeSink::new(querier.as_ref());
ra_parser::parse_ty(&mut token_source, &mut tree_sink);
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
@ -93,9 +96,10 @@ pub fn token_tree_to_macro_stmts(
tt: &tt::Subtree,
) -> Result<TreeArc<ast::MacroStmts>, ExpandError> {
let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
let token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(token_source.querier());
ra_parser::parse_macro_stmts(&token_source, &mut tree_sink);
let mut token_source = SubtreeTokenSource::new(&buffer);
let querier = token_source.querier();
let mut tree_sink = TtTreeSink::new(querier.as_ref());
ra_parser::parse_macro_stmts(&mut token_source, &mut tree_sink);
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
@ -108,9 +112,10 @@ pub fn token_tree_to_macro_items(
tt: &tt::Subtree,
) -> Result<TreeArc<ast::MacroItems>, ExpandError> {
let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
let token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(token_source.querier());
ra_parser::parse_macro_items(&token_source, &mut tree_sink);
let mut token_source = SubtreeTokenSource::new(&buffer);
let querier = token_source.querier();
let mut tree_sink = TtTreeSink::new(querier.as_ref());
ra_parser::parse_macro_items(&mut token_source, &mut tree_sink);
if tree_sink.roots.len() != 1 {
return Err(ExpandError::ConversionError);
}
@ -121,9 +126,10 @@ pub fn token_tree_to_macro_items(
/// Parses the token tree (result of macro expansion) as a sequence of items
pub fn token_tree_to_ast_item_list(tt: &tt::Subtree) -> TreeArc<ast::SourceFile> {
let buffer = tt::buffer::TokenBuffer::new(&[tt.clone().into()]);
let token_source = SubtreeTokenSource::new(&buffer);
let mut tree_sink = TtTreeSink::new(token_source.querier());
ra_parser::parse(&token_source, &mut tree_sink);
let mut token_source = SubtreeTokenSource::new(&buffer);
let querier = token_source.querier();
let mut tree_sink = TtTreeSink::new(querier.as_ref());
ra_parser::parse(&mut token_source, &mut tree_sink);
let syntax = tree_sink.inner.finish();
ast::SourceFile::cast(&syntax).unwrap().to_owned()
}