language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-04 17:40:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Switch parser to use tokens

Browse source
This commit is contained in:
Aleksey Kladov 2021-11-14 22:13:44 +03:00
parent d5ad0f3ca0
commit 26bfd6023f
6 changed files with 47 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

33
crates/parser/src/parser.rs
View file

@ -7,9 +7,10 @@ use limit::Limit;
use crate::{
event::Event,
tokens::Tokens,
ParseError,
SyntaxKind::{self, EOF, ERROR, TOMBSTONE},
TokenSet, TokenSource, T,
TokenSet, T,
};
/// `Parser` struct provides the low-level API for
@ -22,7 +23,8 @@ use crate::{
/// "start expression, consume number literal,
/// finish expression". See `Event` docs for more.
pub(crate) struct Parser<'t> {
token_source: &'t mut dyn TokenSource,
tokens: &'t Tokens,
pos: usize,
events: Vec<Event>,
steps: Cell<u32>,
}
@ -30,8 +32,8 @@ pub(crate) struct Parser<'t> {
static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);
impl<'t> Parser<'t> {
pub(super) fn new(token_source: &'t mut dyn TokenSource) -> Parser<'t> {
Parser { token_source, events: Vec::new(), steps: Cell::new(0) }
pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> {
Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) }
}
pub(crate) fn finish(self) -> Vec<Event> {
@ -54,7 +56,7 @@ impl<'t> Parser<'t> {
assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");
self.steps.set(steps + 1);
self.token_source.lookahead_nth(n).kind
self.tokens.get(self.pos + n).kind
}
/// Checks if the current token is `kind`.
@ -90,7 +92,7 @@ impl<'t> Parser<'t> {
T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),
T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),
_ => self.token_source.lookahead_nth(n).kind == kind,
_ => self.tokens.get(self.pos + n).kind == kind,
}
}
@ -129,24 +131,24 @@ impl<'t> Parser<'t> {
}
fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {
let t1 = self.token_source.lookahead_nth(n);
let t1 = self.tokens.get(self.pos + n);
if t1.kind != k1 || !t1.is_jointed_to_next {
return false;
}
let t2 = self.token_source.lookahead_nth(n + 1);
let t2 = self.tokens.get(self.pos + n + 1);
t2.kind == k2
}
fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {
let t1 = self.token_source.lookahead_nth(n);
let t1 = self.tokens.get(self.pos + n);
if t1.kind != k1 || !t1.is_jointed_to_next {
return false;
}
let t2 = self.token_source.lookahead_nth(n + 1);
let t2 = self.tokens.get(self.pos + n + 1);
if t2.kind != k2 || !t2.is_jointed_to_next {
return false;
}
let t3 = self.token_source.lookahead_nth(n + 2);
let t3 = self.tokens.get(self.pos + n + 2);
t3.kind == k3
}
@ -156,8 +158,8 @@ impl<'t> Parser<'t> {
}
/// Checks if the current token is contextual keyword with text `t`.
pub(crate) fn at_contextual_kw(&self, kw: &str) -> bool {
self.token_source.is_keyword(kw)
pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {
self.tokens.get(self.pos).contextual_kw == kw
}
/// Starts a new node in the syntax tree. All nodes and tokens
@ -243,10 +245,7 @@ impl<'t> Parser<'t> {
}
fn do_bump(&mut self, kind: SyntaxKind, n_raw_tokens: u8) {
for _ in 0..n_raw_tokens {
self.token_source.bump();
}
self.pos += n_raw_tokens as usize;
self.push_event(Event::Token { kind, n_raw_tokens });
}