language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-30 05:45:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Encode ident rawness and literal kind separately in tt::Leaf

Browse source
This commit is contained in:
Lukas Wirth 2024-07-07 17:47:38 +02:00
parent 5784915618
commit e846c04fbe
33 changed files with 860 additions and 412 deletions
Download patch file Download diff file Expand all files Collapse all files

44
crates/hir-expand/src/attrs.rs
View file

@ -5,11 +5,14 @@ use base_db::CrateId;
use cfg::CfgExpr;
use either::Either;
use intern::{sym, Interned};
use mbe::{syntax_node_to_token_tree, DelimiterKind, DocCommentDesugarMode, Punct};
use mbe::{
desugar_doc_comment_text, syntax_node_to_token_tree, token_to_literal, DelimiterKind,
DocCommentDesugarMode, Punct,
};
use smallvec::{smallvec, SmallVec};
use span::{Span, SyntaxContextId};
use syntax::unescape;
use syntax::{ast, format_smolstr, match_ast, AstNode, AstToken, SmolStr, SyntaxNode};
use syntax::{ast, match_ast, AstNode, AstToken, SyntaxNode};
use triomphe::ThinArc;
use crate::name::Name;
@ -53,11 +56,15 @@ impl RawAttrs {
}
Either::Right(comment) => comment.doc_comment().map(|doc| {
let span = span_map.span_for_range(comment.syntax().text_range());
let (text, kind) =
desugar_doc_comment_text(doc, DocCommentDesugarMode::ProcMacro);
Attr {
id,
input: Some(Box::new(AttrInput::Literal(tt::Literal {
text: SmolStr::new(format_smolstr!("\"{}\"", Self::escape_chars(doc))),
text,
span,
kind,
suffix: None,
}))),
path: Interned::new(ModPath::from(Name::new_symbol(
sym::doc.clone(),
@ -78,10 +85,6 @@ impl RawAttrs {
RawAttrs { entries }
}
fn escape_chars(s: &str) -> String {
s.replace('\\', r#"\\"#).replace('"', r#"\""#)
}
pub fn from_attrs_owner(
db: &dyn ExpandDatabase,
owner: InFile<&dyn ast::HasAttrs>,
@ -238,10 +241,8 @@ impl Attr {
})?);
let span = span_map.span_for_range(range);
let input = if let Some(ast::Expr::Literal(lit)) = ast.expr() {
Some(Box::new(AttrInput::Literal(tt::Literal {
text: lit.token().text().into(),
span,
})))
let token = lit.token();
Some(Box::new(AttrInput::Literal(token_to_literal(token.text().into(), span))))
} else if let Some(tt) = ast.token_tree() {
let tree = syntax_node_to_token_tree(
tt.syntax(),
@ -310,12 +311,11 @@ impl Attr {
/// #[path = "string"]
pub fn string_value(&self) -> Option<&str> {
match self.input.as_deref()? {
AttrInput::Literal(it) => match it.text.strip_prefix('r') {
Some(it) => it.trim_matches('#'),
None => it.text.as_str(),
}
.strip_prefix('"')?
.strip_suffix('"'),
AttrInput::Literal(tt::Literal {
text,
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
..
}) => Some(text),
_ => None,
}
}
@ -336,12 +336,10 @@ impl Attr {
pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> {
match self.input.as_deref()? {
AttrInput::Literal(it) => match it.text.strip_prefix('r') {
Some(it) => {
it.trim_matches('#').strip_prefix('"')?.strip_suffix('"').map(Cow::Borrowed)
}
None => it.text.strip_prefix('"')?.strip_suffix('"').and_then(unescape),
},
AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::StrRaw(_), .. }) => {
Some(Cow::Borrowed(text))
}
AttrInput::Literal(tt::Literal { text, kind: tt::LitKind::Str, .. }) => unescape(text),
_ => None,
}
}

3
crates/hir-expand/src/builtin_derive_macro.rs
View file

@ -370,7 +370,8 @@ fn name_to_token(
ExpandError::other("missing name")
})?;
let span = token_map.span_at(name.syntax().text_range().start());
let name_token = tt::Ident { span, text: name.text().into() };
let name_token = tt::Ident::new(name.text().as_ref(), span);
Ok(name_token)
}

199
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -1,13 +1,14 @@
//! Builtin macro
use ::tt::SmolStr;
use base_db::{AnchoredPath, FileId};
use cfg::CfgExpr;
use either::Either;
use intern::sym;
use itertools::Itertools;
use mbe::{parse_exprs_with_sep, parse_to_token_tree};
use span::{Edition, Span, SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID};
use syntax::ast::{self, AstToken};
use stdx::format_to;
use syntax::unescape::{unescape_byte, unescape_char, unescape_unicode, Mode};
use crate::{
db::ExpandDatabase,
@ -177,8 +178,10 @@ fn line_expand(
ExpandResult::ok(tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(span),
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: "0u32".into(),
text: "0".into(),
span,
kind: tt::LitKind::Integer,
suffix: Some(Box::new("u32".into())),
}))]),
})
}
@ -444,27 +447,6 @@ fn use_panic_2021(db: &dyn ExpandDatabase, span: Span) -> bool {
}
}
fn unquote_str(lit: &tt::Literal) -> Option<(String, Span)> {
let span = lit.span;
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::String::cast(lit)?;
token.value().ok().map(|it| (it.into_owned(), span))
}
fn unquote_char(lit: &tt::Literal) -> Option<(char, Span)> {
let span = lit.span;
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::Char::cast(lit)?;
token.value().ok().zip(Some(span))
}
fn unquote_byte_string(lit: &tt::Literal) -> Option<(Vec<u8>, Span)> {
let span = lit.span;
let lit = ast::make::tokens::literal(&lit.to_string());
let token = ast::ByteString::cast(lit)?;
token.value().ok().map(|it| (it.into_owned(), span))
}
fn compile_error_expand(
_db: &dyn ExpandDatabase,
_id: MacroCallId,
@ -472,10 +454,16 @@ fn compile_error_expand(
span: Span,
) -> ExpandResult<tt::Subtree> {
let err = match &*tt.token_trees {
[tt::TokenTree::Leaf(tt::Leaf::Literal(it))] => match unquote_str(it) {
Some((unquoted, _)) => ExpandError::other(unquoted.into_boxed_str()),
None => ExpandError::other("`compile_error!` argument must be a string"),
},
[tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text,
span: _,
kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),
suffix: _,
}))] =>
// FIXME: Use the span here!
{
ExpandError::other(Box::from(&*unescape_str(text)))
}
_ => ExpandError::other("`compile_error!` argument must be a string"),
};
@ -507,20 +495,33 @@ fn concat_expand(
}
}
}
match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(it)) if i % 2 == 0 => {
// concat works with string and char literals, so remove any quotes.
// It also works with integer, float and boolean literals, so just use the rest
// as-is.
if let Some((c, span)) = unquote_char(it) {
text.push(c);
record_span(span);
} else {
let (component, span) =
unquote_str(it).unwrap_or_else(|| (it.text.to_string(), it.span));
text.push_str(&component);
record_span(span);
match it.kind {
tt::LitKind::Char => {
if let Ok(c) = unescape_char(&it.text) {
text.extend(c.escape_default());
}
record_span(it.span);
}
tt::LitKind::Integer | tt::LitKind::Float => format_to!(text, "{}", it.text),
tt::LitKind::Str => {
text.push_str(&it.text);
record_span(it.span);
}
tt::LitKind::StrRaw(_) => {
format_to!(text, "{}", it.text.escape_debug());
record_span(it.span);
}
tt::LitKind::Byte
| tt::LitKind::ByteStr
| tt::LitKind::ByteStrRaw(_)
| tt::LitKind::CStr
| tt::LitKind::CStrRaw(_)
| tt::LitKind::Err(_) => err = Some(ExpandError::other("unexpected literal")),
}
}
// handle boolean literals
@ -544,9 +545,9 @@ fn concat_bytes_expand(
_db: &dyn ExpandDatabase,
_arg_id: MacroCallId,
tt: &tt::Subtree,
call_site: Span,
_: Span,
) -> ExpandResult<tt::Subtree> {
let mut bytes = Vec::new();
let mut bytes = String::new();
let mut err = None;
let mut span: Option<Span> = None;
let mut record_span = |s: Span| match &mut span {
@ -556,14 +557,21 @@ fn concat_bytes_expand(
};
for (i, t) in tt.token_trees.iter().enumerate() {
match t {
tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => {
let token = ast::make::tokens::literal(&lit.to_string());
record_span(lit.span);
match token.kind() {
syntax::SyntaxKind::BYTE => bytes.push(token.text().to_owned()),
syntax::SyntaxKind::BYTE_STRING => {
let components = unquote_byte_string(lit).map_or(vec![], |(it, _)| it);
components.into_iter().for_each(|it| bytes.push(it.to_string()));
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { text, span, kind, suffix: _ })) => {
record_span(*span);
match kind {
tt::LitKind::Byte => {
if let Ok(b) = unescape_byte(text) {
bytes.extend(
b.escape_ascii().filter_map(|it| char::from_u32(it as u32)),
);
}
}
tt::LitKind::ByteStr => {
bytes.push_str(text);
}
tt::LitKind::ByteStrRaw(_) => {
bytes.extend(text.escape_debug());
}
_ => {
err.get_or_insert(mbe::ExpandError::UnexpectedToken.into());
@ -584,51 +592,49 @@ fn concat_bytes_expand(
}
}
}
let value = tt::Subtree {
delimiter: tt::Delimiter {
open: call_site,
close: call_site,
kind: tt::DelimiterKind::Bracket,
let span = span.unwrap_or(tt.delimiter.open);
ExpandResult {
value: tt::Subtree {
delimiter: tt::Delimiter::invisible_spanned(span),
token_trees: vec![tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: bytes.into(),
span,
kind: tt::LitKind::ByteStr,
suffix: None,
}))]
.into(),
},
token_trees: {
Itertools::intersperse_with(
bytes.into_iter().map(|it| {
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: it.into(),
span: span.unwrap_or(call_site),
}))
}),
|| {
tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct {
char: ',',
spacing: tt::Spacing::Alone,
span: call_site,
}))
},
)
.collect()
},
};
ExpandResult { value, err }
err,
}
}
fn concat_bytes_expand_subtree(
tree: &tt::Subtree,
bytes: &mut Vec<String>,
bytes: &mut String,
mut record_span: impl FnMut(Span),
) -> Result<(), ExpandError> {
for (ti, tt) in tree.token_trees.iter().enumerate() {
match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(it)) => {
let lit = ast::make::tokens::literal(&it.to_string());
match lit.kind() {
syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => {
record_span(it.span);
bytes.push(lit.text().to_owned())
}
_ => {
return Err(mbe::ExpandError::UnexpectedToken.into());
}
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text,
span,
kind: tt::LitKind::Byte,
suffix: _,
})) => {
if let Ok(b) = unescape_byte(text) {
bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));
}
record_span(*span);
}
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text,
span,
kind: tt::LitKind::Integer,
suffix: _,
})) => {
record_span(*span);
if let Ok(b) = text.parse::<u8>() {
bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));
}
}
tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if ti % 2 == 1 && punct.char == ',' => (),
@ -660,7 +666,7 @@ fn concat_idents_expand(
}
}
// FIXME merge spans
let ident = tt::Ident { text: ident.into(), span };
let ident = tt::Ident { text: ident.into(), span, is_raw: tt::IdentIsRaw::No };
ExpandResult { value: quote!(span =>#ident), err }
}
@ -683,11 +689,16 @@ fn relative_file(
}
}
fn parse_string(tt: &tt::Subtree) -> Result<(String, Span), ExpandError> {
fn parse_string(tt: &tt::Subtree) -> Result<(SmolStr, Span), ExpandError> {
tt.token_trees
.first()
.and_then(|tt| match tt {
tt::TokenTree::Leaf(tt::Leaf::Literal(it)) => unquote_str(it),
tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text,
span,
kind: tt::LitKind::Str,
suffix: _,
})) => Some((unescape_str(text), *span)),
_ => None,
})
.ok_or(mbe::ExpandError::ConversionError.into())
@ -738,6 +749,8 @@ fn include_bytes_expand(
token_trees: Box::new([tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {
text: r#"b"""#.into(),
span,
kind: tt::LitKind::ByteStrRaw(1),
suffix: None,
}))]),
};
ExpandResult::ok(res)
@ -848,3 +861,17 @@ fn quote_expand(
ExpandError::other("quote! is not implemented"),
)
}
fn unescape_str(s: &SmolStr) -> SmolStr {
if s.contains('\\') {
let mut buf = String::with_capacity(s.len());
unescape_unicode(s, Mode::Str, &mut |_, c| {
if let Ok(c) = c {
buf.push(c)
}
});
buf.into()
} else {
s.clone()
}
}

23
crates/hir-expand/src/fixup.rs
View file

@ -86,6 +86,7 @@ pub(crate) fn fixup_syntax(
anchor: SpanAnchor { ast_id: FIXUP_DUMMY_AST_ID, ..span.anchor },
ctx: span.ctx,
},
is_raw: tt::IdentIsRaw::No,
});
append.insert(node.clone().into(), vec![replacement]);
preorder.skip_subtree();
@ -101,6 +102,7 @@ pub(crate) fn fixup_syntax(
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
}),
]);
}
@ -137,7 +139,8 @@ pub(crate) fn fixup_syntax(
append.insert(if_token.into(), vec![
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
}),
]);
}
@ -167,7 +170,8 @@ pub(crate) fn fixup_syntax(
append.insert(while_token.into(), vec![
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
}),
]);
}
@ -214,7 +218,8 @@ pub(crate) fn fixup_syntax(
append.insert(match_token.into(), vec![
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
}),
]);
}
@ -248,7 +253,8 @@ pub(crate) fn fixup_syntax(
].map(|text|
Leaf::Ident(Ident {
text: text.into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
}),
);
@ -281,7 +287,8 @@ pub(crate) fn fixup_syntax(
append.insert(colon.into(), vec![
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
})
]);
}
@ -293,7 +300,8 @@ pub(crate) fn fixup_syntax(
append.insert(colon.into(), vec![
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
})
]);
}
@ -326,7 +334,8 @@ pub(crate) fn fixup_syntax(
append.insert(node.into(), vec![
Leaf::Ident(Ident {
text: "__ra_fixup".into(),
span: fake_span(node_range)
span: fake_span(node_range),
is_raw: tt::IdentIsRaw::No
})
]);
}

2
crates/hir-expand/src/lib.rs
View file

@ -59,7 +59,7 @@ pub use span::{HirFileId, MacroCallId, MacroFileId};
pub mod tt {
pub use span::Span;
pub use tt::{DelimiterKind, Spacing};
pub use tt::{DelimiterKind, IdentIsRaw, LitKind, Spacing};
pub type Delimiter = ::tt::Delimiter<Span>;
pub type DelimSpan = ::tt::DelimSpan<Span>;

10
crates/hir-expand/src/mod_path.rs
View file

@ -316,15 +316,15 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: &[tt::TokenTree]) -> Option<ModP
tt::Leaf::Punct(tt::Punct { char: ':', .. }) => PathKind::Abs,
_ => return None,
},
tt::Leaf::Ident(tt::Ident { text, span }) if text == "$crate" => {
tt::Leaf::Ident(tt::Ident { text, span, .. }) if text == "$crate" => {
resolve_crate_root(db, span.ctx).map(PathKind::DollarCrate).unwrap_or(PathKind::Crate)
}
tt::Leaf::Ident(tt::Ident { text, .. }) if text == "self" => PathKind::SELF,
tt::Leaf::Ident(tt::Ident { text, .. }) if text == "super" => {
let mut deg = 1;
while let Some(tt::Leaf::Ident(tt::Ident { text, span, .. })) = leaves.next() {
while let Some(tt::Leaf::Ident(tt::Ident { text, span, is_raw })) = leaves.next() {
if text != "super" {
segments.push(Name::new(text, span.ctx));
segments.push(Name::new(text, *is_raw, span.ctx));
break;
}
deg += 1;
@ -333,13 +333,13 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: &[tt::TokenTree]) -> Option<ModP
}
tt::Leaf::Ident(tt::Ident { text, .. }) if text == "crate" => PathKind::Crate,
tt::Leaf::Ident(ident) => {
segments.push(Name::new(&ident.text, ident.span.ctx));
segments.push(Name::new(&ident.text, ident.is_raw, ident.span.ctx));
PathKind::Plain
}
_ => return None,
};
segments.extend(leaves.filter_map(|leaf| match leaf {
::tt::Leaf::Ident(ident) => Some(Name::new(&ident.text, ident.span.ctx)),
::tt::Leaf::Ident(ident) => Some(Name::new(&ident.text, ident.is_raw, ident.span.ctx)),
_ => None,
}));
Some(ModPath { kind, segments })

11
crates/hir-expand/src/name.rs
View file

@ -82,9 +82,16 @@ impl Name {
Name { symbol: Symbol::intern(text), ctx: () }
}
pub fn new(text: &str, ctx: SyntaxContextId) -> Name {
pub fn new(text: &str, raw: tt::IdentIsRaw, ctx: SyntaxContextId) -> Name {
_ = ctx;
Name { symbol: Symbol::intern(text), ctx: () }
Name {
symbol: if raw.yes() {
Symbol::intern(&format_smolstr!("{}{text}", raw.as_str()))
} else {
Symbol::intern(text)
},
ctx: (),
}
}
pub fn new_tuple_field(idx: usize) -> Name {

31
crates/hir-expand/src/quote.rs
View file

@ -3,12 +3,12 @@
use intern::Symbol;
use span::Span;
use syntax::format_smolstr;
use tt::IdentIsRaw;
use crate::name::Name;
pub(crate) const fn dollar_crate(span: Span) -> tt::Ident<Span> {
tt::Ident { text: syntax::SmolStr::new_static("$crate"), span }
tt::Ident { text: syntax::SmolStr::new_static("$crate"), span, is_raw: tt::IdentIsRaw::No }
}
// A helper macro quote macro
@ -101,6 +101,7 @@ macro_rules! __quote {
crate::tt::Leaf::Ident(crate::tt::Ident {
text: stringify!($tt).into(),
span: $span,
is_raw: tt::IdentIsRaw::No,
}).into()
}]
};
@ -209,23 +210,30 @@ macro_rules! impl_to_to_tokentrees {
}
impl_to_to_tokentrees! {
span: u32 => self { crate::tt::Literal{text: self.to_string().into(), span} };
span: usize => self { crate::tt::Literal{text: self.to_string().into(), span} };
span: i32 => self { crate::tt::Literal{text: self.to_string().into(), span} };
span: bool => self { crate::tt::Ident{text: self.to_string().into(), span} };
span: u32 => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer, suffix: None } };
span: usize => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer, suffix: None } };
span: i32 => self { crate::tt::Literal{text: self.to_string().into(), span, kind: tt::LitKind::Integer, suffix: None } };
span: bool => self { crate::tt::Ident{text: self.to_string().into(), span, is_raw: tt::IdentIsRaw::No } };
_span: crate::tt::Leaf => self { self };
_span: crate::tt::Literal => self { self };
_span: crate::tt::Ident => self { self };
_span: crate::tt::Punct => self { self };
span: &str => self { crate::tt::Literal{text: format_smolstr!("\"{}\"", self.escape_default()), span}};
span: String => self { crate::tt::Literal{text: format_smolstr!("\"{}\"", self.escape_default()), span}};
span: Name => self { crate::tt::Ident{text: self.to_smol_str(), span}};
span: Symbol => self { crate::tt::Ident{text: self.as_str().into(), span}};
span: &str => self { crate::tt::Literal{text: (*self).into(), span, kind: tt::LitKind::Str, suffix: None }};
span: String => self { crate::tt::Literal{text: self.into(), span, kind: tt::LitKind::Str, suffix: None }};
span: Name => self {
let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str());
crate::tt::Ident{text: s.into(), span, is_raw }
};
span: Symbol => self {
let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str());
crate::tt::Ident{text: s.into(), span, is_raw }
};
}
#[cfg(test)]
mod tests {
use crate::tt;
use ::tt::IdentIsRaw;
use base_db::FileId;
use expect_test::expect;
use span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID};
@ -259,7 +267,8 @@ mod tests {
}
fn mk_ident(name: &str) -> crate::tt::Ident {
crate::tt::Ident { text: name.into(), span: DUMMY }
let (is_raw, s) = IdentIsRaw::split_from_symbol(name);
crate::tt::Ident { text: s.into(), span: DUMMY, is_raw }
}
#[test]