language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-30 05:45:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Split out syntax-bridge into a separate crate

Browse source
This commit is contained in:
Lukas Wirth 2024-08-05 10:43:01 +02:00
parent 670a5ab4a9
commit d2dd4f6d5f
30 changed files with 268 additions and 140 deletions
Download patch file Download diff file Expand all files Collapse all files

6
crates/mbe/src/benchmark.rs
View file

@ -7,11 +7,15 @@ use syntax::{
ast::{self, HasName},
AstNode,
};
use syntax_bridge::{
dummy_test_span_utils::{DummyTestSpanMap, DUMMY},
syntax_node_to_token_tree, DocCommentDesugarMode,
};
use test_utils::{bench, bench_fixture, skip_slow_tests};
use crate::{
parser::{MetaVarKind, Op, RepeatKind, Separator},
syntax_node_to_token_tree, DeclarativeMacro, DocCommentDesugarMode, DummyTestSpanMap, DUMMY,
DeclarativeMacro,
};
#[test]

17
crates/mbe/src/lib.rs
View file

@ -8,13 +8,12 @@
mod expander;
mod parser;
mod syntax_bridge;
mod to_parser_input;
#[cfg(test)]
mod benchmark;
use span::{Edition, Span, SyntaxContextId};
use syntax_bridge::to_parser_input;
use tt::iter::TtIter;
use tt::DelimSpan;
@ -23,18 +22,8 @@ use std::sync::Arc;
use crate::parser::{MetaTemplate, MetaVarKind, Op};
// FIXME: we probably should re-think `token_tree_to_syntax_node` interfaces
pub use ::parser::TopEntryPoint;
pub use tt::{Delimiter, DelimiterKind, Punct};
pub use crate::syntax_bridge::{
desugar_doc_comment_text, parse_exprs_with_sep, parse_to_token_tree,
parse_to_token_tree_static_span, syntax_node_to_token_tree, syntax_node_to_token_tree_modified,
token_tree_to_syntax_node, DocCommentDesugarMode, SpanMapper,
};
pub use crate::syntax_bridge::dummy_test_span_utils::*;
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum ParseError {
UnexpectedToken(Box<str>),
@ -361,7 +350,7 @@ impl<T: Default, E> From<Result<T, E>> for ValueResult<T, E> {
}
}
fn expect_fragment(
pub fn expect_fragment(
tt_iter: &mut TtIter<'_, Span>,
entry_point: ::parser::PrefixEntryPoint,
edition: ::parser::Edition,
@ -369,7 +358,7 @@ fn expect_fragment(
) -> ExpandResult<Option<tt::TokenTree<Span>>> {
use ::parser;
let buffer = tt::buffer::TokenBuffer::from_tokens(tt_iter.as_slice());
let parser_input = to_parser_input::to_parser_input(edition, &buffer);
let parser_input = to_parser_input(edition, &buffer);
let tree_traversal = entry_point.parse(&parser_input, edition);
let mut cursor = buffer.begin();
let mut error = false;

1090
crates/mbe/src/syntax_bridge.rs
View file

File diff suppressed because it is too large Load diff

119
crates/mbe/src/to_parser_input.rs
View file

@ -1,119 +0,0 @@
//! Convert macro-by-example tokens which are specific to macro expansion into a
//! format that works for our parser.
use std::fmt;
use span::Edition;
use syntax::{SyntaxKind, SyntaxKind::*, T};
use tt::buffer::TokenBuffer;
pub(crate) fn to_parser_input<S: Copy + fmt::Debug>(
edition: Edition,
buffer: &TokenBuffer<'_, S>,
) -> parser::Input {
let mut res = parser::Input::default();
let mut current = buffer.begin();
while !current.eof() {
let cursor = current;
let tt = cursor.token_tree();
// Check if it is lifetime
if let Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(punct), _)) = tt {
if punct.char == '\'' {
let next = cursor.bump();
match next.token_tree() {
Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Ident(_ident), _)) => {
res.push(LIFETIME_IDENT);
current = next.bump();
continue;
}
_ => panic!("Next token must be ident : {:#?}", next.token_tree()),
}
}
}
current = match tt {
Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => {
match leaf {
tt::Leaf::Literal(lit) => {
let kind = match lit.kind {
tt::LitKind::Byte => SyntaxKind::BYTE,
tt::LitKind::Char => SyntaxKind::CHAR,
tt::LitKind::Integer => SyntaxKind::INT_NUMBER,
tt::LitKind::Float => SyntaxKind::FLOAT_NUMBER,
tt::LitKind::Str | tt::LitKind::StrRaw(_) => SyntaxKind::STRING,
tt::LitKind::ByteStr | tt::LitKind::ByteStrRaw(_) => {
SyntaxKind::BYTE_STRING
}
tt::LitKind::CStr | tt::LitKind::CStrRaw(_) => SyntaxKind::C_STRING,
tt::LitKind::Err(_) => SyntaxKind::ERROR,
};
res.push(kind);
if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') {
// Tag the token as joint if it is float with a fractional part
// we use this jointness to inform the parser about what token split
// event to emit when we encounter a float literal in a field access
res.was_joint();
}
}
tt::Leaf::Ident(ident) => match ident.sym.as_str() {
"_" => res.push(T![_]),
i if i.starts_with('\'') => res.push(LIFETIME_IDENT),
_ if ident.is_raw.yes() => res.push(IDENT),
"gen" if !edition.at_least_2024() => res.push(IDENT),
"dyn" if !edition.at_least_2018() => res.push_ident(DYN_KW),
"async" | "await" | "try" if !edition.at_least_2018() => res.push(IDENT),
text => match SyntaxKind::from_keyword(text) {
Some(kind) => res.push(kind),
None => {
let contextual_keyword = SyntaxKind::from_contextual_keyword(text)
.unwrap_or(SyntaxKind::IDENT);
res.push_ident(contextual_keyword);
}
},
},
tt::Leaf::Punct(punct) => {
let kind = SyntaxKind::from_char(punct.char)
.unwrap_or_else(|| panic!("{punct:#?} is not a valid punct"));
res.push(kind);
if punct.spacing == tt::Spacing::Joint {
res.was_joint();
}
}
}
cursor.bump()
}
Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => {
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T!['(']),
tt::DelimiterKind::Brace => Some(T!['{']),
tt::DelimiterKind::Bracket => Some(T!['[']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.subtree().unwrap()
}
None => match cursor.end() {
Some(subtree) => {
if let Some(kind) = match subtree.delimiter.kind {
tt::DelimiterKind::Parenthesis => Some(T![')']),
tt::DelimiterKind::Brace => Some(T!['}']),
tt::DelimiterKind::Bracket => Some(T![']']),
tt::DelimiterKind::Invisible => None,
} {
res.push(kind);
}
cursor.bump()
}
None => continue,
},
};
}
res
}