language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-28 21:05:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

spans always come from real file

Browse source
This commit is contained in:
Lukas Wirth 2023-11-24 16:38:48 +01:00
parent 394d11b0fa
commit 30093a6d81
57 changed files with 1369 additions and 1224 deletions
Download patch file Download diff file Expand all files Collapse all files

47
crates/hir-expand/src/ast_id_map.rs
View file

@ -19,6 +19,33 @@ use syntax::{ast, AstNode, AstPtr, SyntaxNode, SyntaxNodePtr};
pub use base_db::span::ErasedFileAstId;
use crate::db;
/// `AstId` points to an AST node in any file.
///
/// It is stable across reparses, and can be used as salsa key/value.
pub type AstId<N> = crate::InFile<FileAstId<N>>;
impl<N: AstIdNode> AstId<N> {
pub fn to_node(&self, db: &dyn db::ExpandDatabase) -> N {
self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id))
}
pub fn to_in_file_node(&self, db: &dyn db::ExpandDatabase) -> crate::InFile<N> {
crate::InFile::new(self.file_id, self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id)))
}
pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> AstPtr<N> {
db.ast_id_map(self.file_id).get(self.value)
}
}
pub type ErasedAstId = crate::InFile<ErasedFileAstId>;
impl ErasedAstId {
pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> SyntaxNodePtr {
db.ast_id_map(self.file_id).get_erased(self.value)
}
}
/// `AstId` points to an AST node in a specific file.
pub struct FileAstId<N: AstIdNode> {
raw: ErasedFileAstId,
@ -141,9 +168,9 @@ impl AstIdMap {
bdfs(node, |it| {
if should_alloc_id(it.kind()) {
res.alloc(&it);
true
TreeOrder::BreadthFirst
} else {
false
TreeOrder::DepthFirst
}
});
res.map = hashbrown::HashMap::with_capacity_and_hasher(res.arena.len(), ());
@ -174,7 +201,7 @@ impl AstIdMap {
AstPtr::try_from_raw(self.arena[id.raw].clone()).unwrap()
}
pub fn get_raw(&self, id: ErasedFileAstId) -> SyntaxNodePtr {
pub fn get_erased(&self, id: ErasedFileAstId) -> SyntaxNodePtr {
self.arena[id].clone()
}
@ -202,14 +229,20 @@ fn hash_ptr(ptr: &SyntaxNodePtr) -> u64 {
hasher.finish()
}
#[derive(Copy, Clone, PartialEq, Eq)]
enum TreeOrder {
BreadthFirst,
DepthFirst,
}
/// Walks the subtree in bdfs order, calling `f` for each node. What is bdfs
/// order? It is a mix of breadth-first and depth first orders. Nodes for which
/// `f` returns true are visited breadth-first, all the other nodes are explored
/// depth-first.
/// `f` returns [`TreeOrder::BreadthFirst`] are visited breadth-first, all the other nodes are explored
/// [`TreeOrder::DepthFirst`].
///
/// In other words, the size of the bfs queue is bound by the number of "true"
/// nodes.
fn bdfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode) -> bool) {
fn bdfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode) -> TreeOrder) {
let mut curr_layer = vec![node.clone()];
let mut next_layer = vec![];
while !curr_layer.is_empty() {
@ -218,7 +251,7 @@ fn bdfs(node: &SyntaxNode, mut f: impl FnMut(SyntaxNode) -> bool) {
while let Some(event) = preorder.next() {
match event {
syntax::WalkEvent::Enter(node) => {
if f(node.clone()) {
if f(node.clone()) == TreeOrder::BreadthFirst {
next_layer.extend(node.children());
preorder.skip_subtree();
}

49
crates/hir-expand/src/attrs.rs
View file

@ -1,11 +1,7 @@
//! A higher level attributes based on TokenTree, with also some shortcuts.
use std::{fmt, ops};
use ::tt::SpanAnchor as _;
use base_db::{
span::{SpanAnchor, SyntaxContextId},
CrateId,
};
use base_db::{span::SyntaxContextId, CrateId};
use cfg::CfgExpr;
use either::Either;
use intern::Interned;
@ -17,8 +13,9 @@ use triomphe::Arc;
use crate::{
db::ExpandDatabase,
mod_path::ModPath,
span::SpanMapRef,
tt::{self, Subtree},
InFile, SpanMap,
InFile,
};
/// Syntactical attributes, without filtering of `cfg_attr`s.
@ -44,22 +41,19 @@ impl RawAttrs {
pub fn new(
db: &dyn ExpandDatabase,
span_anchor: SpanAnchor,
owner: &dyn ast::HasAttrs,
hygiene: &SpanMap,
hygiene: SpanMapRef<'_>,
) -> Self {
let entries = collect_attrs(owner)
.filter_map(|(id, attr)| match attr {
Either::Left(attr) => {
attr.meta().and_then(|meta| Attr::from_src(db, span_anchor, meta, hygiene, id))
attr.meta().and_then(|meta| Attr::from_src(db, meta, hygiene, id))
}
Either::Right(comment) => comment.doc_comment().map(|doc| Attr {
id,
input: Some(Interned::new(AttrInput::Literal(SmolStr::new(doc)))),
path: Interned::new(ModPath::from(crate::name!(doc))),
ctxt: hygiene
.span_for_range(comment.syntax().text_range())
.map_or(SyntaxContextId::ROOT, |s| s.ctx),
ctxt: hygiene.span_for_range(comment.syntax().text_range()).ctx,
}),
})
.collect::<Vec<_>>();
@ -71,10 +65,10 @@ impl RawAttrs {
pub fn from_attrs_owner(
db: &dyn ExpandDatabase,
span_anchor: SpanAnchor,
owner: InFile<&dyn ast::HasAttrs>,
hygiene: SpanMapRef<'_>,
) -> Self {
Self::new(db, span_anchor, owner.value, &db.span_map(owner.file_id))
Self::new(db, owner.value, hygiene)
}
pub fn merge(&self, other: Self) -> Self {
@ -221,9 +215,8 @@ impl fmt::Display for AttrInput {
impl Attr {
fn from_src(
db: &dyn ExpandDatabase,
span_anchor: SpanAnchor,
ast: ast::Meta,
hygiene: &SpanMap,
hygiene: SpanMapRef<'_>,
id: AttrId,
) -> Option<Attr> {
let path = Interned::new(ModPath::from_src(db, ast.path()?, hygiene)?);
@ -234,31 +227,20 @@ impl Attr {
};
Some(Interned::new(AttrInput::Literal(value)))
} else if let Some(tt) = ast.token_tree() {
// FIXME: We could also allocate ids for attributes and use the attribute itself as an anchor
let offset =
db.ast_id_map(span_anchor.file_id).get_raw(span_anchor.ast_id).text_range().start();
let tree = syntax_node_to_token_tree(tt.syntax(), span_anchor, offset, hygiene);
let tree = syntax_node_to_token_tree(tt.syntax(), hygiene);
Some(Interned::new(AttrInput::TokenTree(Box::new(tree))))
} else {
None
};
Some(Attr {
id,
path,
input,
ctxt: hygiene
.span_for_range(ast.syntax().text_range())
.map_or(SyntaxContextId::ROOT, |s| s.ctx),
})
Some(Attr { id, path, input, ctxt: hygiene.span_for_range(ast.syntax().text_range()).ctx })
}
fn from_tt(db: &dyn ExpandDatabase, tt: &tt::Subtree, id: AttrId) -> Option<Attr> {
// FIXME: Unecessary roundtrip tt -> ast -> tt
let (parse, _map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem);
let (parse, map) = mbe::token_tree_to_syntax_node(tt, mbe::TopEntryPoint::MetaItem);
let ast = ast::Meta::cast(parse.syntax_node())?;
// FIXME: we discard spans here!
Self::from_src(db, SpanAnchor::DUMMY, ast, &SpanMap::default(), id)
Self::from_src(db, ast, SpanMapRef::ExpansionSpanMap(&map), id)
}
pub fn path(&self) -> &ModPath {
@ -331,7 +313,10 @@ impl Attr {
return None;
}
let path = meta.path()?;
Some((ModPath::from_src(db, path, &span_map)?, call_site))
Some((
ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(&span_map))?,
call_site,
))
});
Some(paths)

9
crates/hir-expand/src/builtin_attr_macro.rs
View file

@ -79,9 +79,8 @@ fn dummy_attr_expand(
///
/// As such, we expand `#[derive(Foo, bar::Bar)]` into
/// ```
/// #[Foo]
/// #[bar::Bar]
/// ();
/// #![Foo]
/// #![bar::Bar]
/// ```
/// which allows fallback path resolution in hir::Semantics to properly identify our derives.
/// Since we do not expand the attribute in nameres though, we keep the original item.
@ -124,12 +123,10 @@ pub fn pseudo_derive_attr_expansion(
.split(|tt| matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: ',', .. }))))
{
token_trees.push(mk_leaf('#'));
token_trees.push(mk_leaf('!'));
token_trees.push(mk_leaf('['));
token_trees.extend(tt.iter().cloned());
token_trees.push(mk_leaf(']'));
}
token_trees.push(mk_leaf('('));
token_trees.push(mk_leaf(')'));
token_trees.push(mk_leaf(';'));
ExpandResult::ok(tt::Subtree { delimiter: tt.delimiter, token_trees })
}

122
crates/hir-expand/src/builtin_derive_macro.rs
View file

@ -1,20 +1,19 @@
//! Builtin derives.
use ::tt::Span;
use base_db::{CrateOrigin, LangCrateOrigin};
use base_db::{span::SpanData, CrateOrigin, LangCrateOrigin};
use itertools::izip;
use rustc_hash::FxHashSet;
use stdx::never;
use tracing::debug;
use crate::{
hygiene::span_with_def_site_ctxt,
name::{AsName, Name},
tt, SpanMap,
};
use syntax::{
ast::{self, AstNode, FieldList, HasAttrs, HasGenericParams, HasName, HasTypeBounds},
TextSize,
span::SpanMapRef,
tt,
};
use syntax::ast::{self, AstNode, FieldList, HasAttrs, HasGenericParams, HasName, HasTypeBounds};
use crate::{db::ExpandDatabase, name, quote, ExpandError, ExpandResult, MacroCallId};
@ -31,12 +30,15 @@ macro_rules! register_builtin {
db: &dyn ExpandDatabase,
id: MacroCallId,
tt: &ast::Adt,
token_map: &SpanMap,
token_map: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let expander = match *self {
$( BuiltinDeriveExpander::$trait => $expand, )*
};
expander(db, id, tt, token_map)
let span = db.lookup_intern_macro_call(id).span(db);
let span = span_with_def_site_ctxt(db, span, id);
expander(db, id, span, tt, token_map)
}
fn find_by_name(name: &name::Name) -> Option<Self> {
@ -119,7 +121,7 @@ impl VariantShape {
}
}
fn from(tm: &SpanMap, value: Option<FieldList>) -> Result<Self, ExpandError> {
fn from(tm: SpanMapRef<'_>, value: Option<FieldList>) -> Result<Self, ExpandError> {
let r = match value {
None => VariantShape::Unit,
Some(FieldList::RecordFieldList(it)) => VariantShape::Struct(
@ -191,7 +193,7 @@ struct BasicAdtInfo {
associated_types: Vec<tt::Subtree>,
}
fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result<BasicAdtInfo, ExpandError> {
fn parse_adt(tm: SpanMapRef<'_>, adt: &ast::Adt) -> Result<BasicAdtInfo, ExpandError> {
let (name, generic_param_list, shape) = match adt {
ast::Adt::Struct(it) => (
it.name(),
@ -236,44 +238,21 @@ fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result<BasicAdtInfo, ExpandError>
match this {
Some(it) => {
param_type_set.insert(it.as_name());
mbe::syntax_node_to_token_tree(
it.syntax(),
tm.span_for_range(it.syntax().first_token().unwrap().text_range())
.unwrap()
.anchor,
TextSize::from(0),
tm,
)
mbe::syntax_node_to_token_tree(it.syntax(), tm)
}
None => tt::Subtree::empty(),
}
};
let bounds = match &param {
ast::TypeOrConstParam::Type(it) => it.type_bound_list().map(|it| {
mbe::syntax_node_to_token_tree(
it.syntax(),
tm.span_for_range(it.syntax().first_token().unwrap().text_range())
.unwrap()
.anchor,
TextSize::from(0),
tm,
)
}),
ast::TypeOrConstParam::Type(it) => {
it.type_bound_list().map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm))
}
ast::TypeOrConstParam::Const(_) => None,
};
let ty = if let ast::TypeOrConstParam::Const(param) = param {
let ty = param
.ty()
.map(|ty| {
mbe::syntax_node_to_token_tree(
ty.syntax(),
tm.span_for_range(ty.syntax().first_token().unwrap().text_range())
.unwrap()
.anchor,
TextSize::from(0),
tm,
)
})
.map(|ty| mbe::syntax_node_to_token_tree(ty.syntax(), tm))
.unwrap_or_else(tt::Subtree::empty);
Some(ty)
} else {
@ -307,25 +286,21 @@ fn parse_adt(tm: &SpanMap, adt: &ast::Adt) -> Result<BasicAdtInfo, ExpandError>
let name = p.path()?.qualifier()?.as_single_name_ref()?.as_name();
param_type_set.contains(&name).then_some(p)
})
.map(|it| {
mbe::syntax_node_to_token_tree(
it.syntax(),
tm.span_for_range(it.syntax().first_token().unwrap().text_range()).unwrap().anchor,
TextSize::from(0),
tm,
)
})
.map(|it| mbe::syntax_node_to_token_tree(it.syntax(), tm))
.collect();
let name_token = name_to_token(&tm, name)?;
let name_token = name_to_token(tm, name)?;
Ok(BasicAdtInfo { name: name_token, shape, param_types, associated_types })
}
fn name_to_token(token_map: &SpanMap, name: Option<ast::Name>) -> Result<tt::Ident, ExpandError> {
fn name_to_token(
token_map: SpanMapRef<'_>,
name: Option<ast::Name>,
) -> Result<tt::Ident, ExpandError> {
let name = name.ok_or_else(|| {
debug!("parsed item has no name");
ExpandError::other("missing name")
})?;
let span = token_map.span_for_range(name.syntax().text_range()).unwrap();
let span = token_map.span_for_range(name.syntax().text_range());
let name_token = tt::Ident { span, text: name.text().into() };
Ok(name_token)
}
@ -362,8 +337,10 @@ fn name_to_token(token_map: &SpanMap, name: Option<ast::Name>) -> Result<tt::Ide
/// where B1, ..., BN are the bounds given by `bounds_paths`. Z is a phantom type, and
/// therefore does not get bound by the derived trait.
fn expand_simple_derive(
// FIXME: use
_invoc_span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
trait_path: tt::Subtree,
make_trait_body: impl FnOnce(&BasicAdtInfo) -> tt::Subtree,
) -> ExpandResult<tt::Subtree> {
@ -423,21 +400,23 @@ fn find_builtin_crate(db: &dyn ExpandDatabase, id: MacroCallId) -> tt::TokenTree
fn copy_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::marker::Copy }, |_| quote! {})
expand_simple_derive(span, tt, tm, quote! { #krate::marker::Copy }, |_| quote! {})
}
fn clone_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::clone::Clone }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::clone::Clone }, |adt| {
if matches!(adt.shape, AdtShape::Union) {
let star =
tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span: tt::SpanData::DUMMY };
@ -491,11 +470,12 @@ fn and_and() -> tt::Subtree {
fn default_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = &find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::default::Default }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::default::Default }, |adt| {
let body = match &adt.shape {
AdtShape::Struct(fields) => {
let name = &adt.name;
@ -531,11 +511,12 @@ fn default_expand(
fn debug_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = &find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::fmt::Debug }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::fmt::Debug }, |adt| {
let for_variant = |name: String, v: &VariantShape| match v {
VariantShape::Struct(fields) => {
let for_fields = fields.iter().map(|it| {
@ -609,11 +590,12 @@ fn debug_expand(
fn hash_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = &find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::hash::Hash }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::hash::Hash }, |adt| {
if matches!(adt.shape, AdtShape::Union) {
// FIXME: Return expand error here
return quote! {};
@ -660,21 +642,23 @@ fn hash_expand(
fn eq_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::cmp::Eq }, |_| quote! {})
expand_simple_derive(span, tt, tm, quote! { #krate::cmp::Eq }, |_| quote! {})
}
fn partial_eq_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::cmp::PartialEq }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::cmp::PartialEq }, |adt| {
if matches!(adt.shape, AdtShape::Union) {
// FIXME: Return expand error here
return quote! {};
@ -738,11 +722,12 @@ fn self_and_other_patterns(
fn ord_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = &find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::cmp::Ord }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::cmp::Ord }, |adt| {
fn compare(
krate: &tt::TokenTree,
left: tt::Subtree,
@ -800,11 +785,12 @@ fn ord_expand(
fn partial_ord_expand(
db: &dyn ExpandDatabase,
id: MacroCallId,
span: SpanData,
tt: &ast::Adt,
tm: &SpanMap,
tm: SpanMapRef<'_>,
) -> ExpandResult<tt::Subtree> {
let krate = &find_builtin_crate(db, id);
expand_simple_derive(tt, tm, quote! { #krate::cmp::PartialOrd }, |adt| {
expand_simple_derive(span, tt, tm, quote! { #krate::cmp::PartialOrd }, |adt| {
fn compare(
krate: &tt::TokenTree,
left: tt::Subtree,

3
crates/hir-expand/src/builtin_fn_macro.rs
View file

@ -556,9 +556,10 @@ pub(crate) fn include_arg_to_tt(
let path = parse_string(&arg.0)?;
let file_id = relative_file(db, *arg_id, &path, false)?;
// why are we not going through a SyntaxNode here?
let subtree = parse_to_token_tree(
SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
&db.file_text(file_id),
SpanAnchor { file_id: file_id.into(), ast_id: ROOT_ERASED_FILE_AST_ID },
)
.ok_or(mbe::ExpandError::ConversionError)?;
Ok((triomphe::Arc::new(subtree), file_id))

152
crates/hir-expand/src/db.rs
View file

@ -3,15 +3,15 @@
use ::tt::{SpanAnchor as _, SyntaxContext};
use base_db::{
salsa,
span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID},
CrateId, Edition, SourceDatabase,
span::{SpanAnchor, SyntaxContextId},
CrateId, Edition, FileId, SourceDatabase,
};
use either::Either;
use limit::Limit;
use mbe::{map_from_syntax_node, syntax_node_to_token_tree, ValueResult};
use mbe::{syntax_node_to_token_tree, ValueResult};
use syntax::{
ast::{self, HasAttrs, HasDocComments},
AstNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, TextSize, T,
AstNode, Parse, SyntaxError, SyntaxNode, SyntaxToken, T,
};
use triomphe::Arc;
@ -21,9 +21,10 @@ use crate::{
builtin_attr_macro::pseudo_derive_attr_expansion,
builtin_fn_macro::EagerExpander,
hygiene::{self, SyntaxContextData, Transparency},
span::{RealSpanMap, SpanMap, SpanMapRef},
tt, AstId, BuiltinAttrExpander, BuiltinDeriveExpander, BuiltinFnLikeExpander, EagerCallInfo,
ExpandError, ExpandResult, ExpandTo, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind,
MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, SpanMap,
ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, HirFileId, HirFileIdRepr, MacroCallId,
MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander,
};
/// Total limit on the number of tokens produced by any macro invocation.
@ -102,10 +103,11 @@ pub trait ExpandDatabase: SourceDatabase {
fn parse_macro_expansion(
&self,
macro_file: MacroFile,
) -> ExpandResult<(Parse<SyntaxNode>, Arc<SpanMap>)>;
// FIXME: This always allocates one for non macro files which is wasteful.
) -> ExpandResult<(Parse<SyntaxNode>, Arc<ExpansionSpanMap>)>;
#[salsa::transparent]
fn span_map(&self, file_id: HirFileId) -> Arc<SpanMap>;
fn span_map(&self, file_id: HirFileId) -> SpanMap;
fn real_span_map(&self, file_id: FileId) -> Arc<RealSpanMap>;
/// Macro ids. That's probably the tricksiest bit in rust-analyzer, and the
/// reason why we use salsa at all.
@ -164,13 +166,20 @@ pub trait ExpandDatabase: SourceDatabase {
) -> ExpandResult<Box<[SyntaxError]>>;
}
fn span_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> Arc<SpanMap> {
#[inline]
pub fn span_map(db: &dyn ExpandDatabase, file_id: HirFileId) -> SpanMap {
match file_id.repr() {
HirFileIdRepr::FileId(_) => Arc::new(Default::default()),
HirFileIdRepr::MacroFile(m) => db.parse_macro_expansion(m).value.1,
HirFileIdRepr::FileId(file_id) => SpanMap::RealSpanMap(db.real_span_map(file_id)),
HirFileIdRepr::MacroFile(m) => {
SpanMap::ExpansionSpanMap(db.parse_macro_expansion(m).value.1)
}
}
}
pub fn real_span_map(db: &dyn ExpandDatabase, file_id: FileId) -> Arc<RealSpanMap> {
Arc::new(RealSpanMap::from_file(db, file_id))
}
/// This expands the given macro call, but with different arguments. This is
/// used for completion, where we want to see what 'would happen' if we insert a
/// token. The `token_to_map` mapped down into the expansion, with the mapped
@ -181,17 +190,15 @@ pub fn expand_speculative(
speculative_args: &SyntaxNode,
token_to_map: SyntaxToken,
) -> Option<(SyntaxNode, SyntaxToken)> {
// FIXME spanmaps
let loc = db.lookup_intern_macro_call(actual_macro_call);
let file_id = loc.kind.file_id();
// Build the subtree and token mapping for the speculative args
let _censor = censor_for_macro_input(&loc, speculative_args);
let mut tt = mbe::syntax_node_to_token_tree(
speculative_args,
// we don't leak these spans into any query so its fine to make them absolute
SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
TextSize::new(0),
&Default::default(),
SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)),
);
let attr_arg = match loc.kind {
@ -211,9 +218,7 @@ pub fn expand_speculative(
Some(token_tree) => {
let mut tree = syntax_node_to_token_tree(
token_tree.syntax(),
SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
TextSize::new(0),
&Default::default(),
SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)),
);
tree.delimiter = tt::Delimiter::UNSPECIFIED;
@ -242,12 +247,7 @@ pub fn expand_speculative(
db,
actual_macro_call,
&adt,
&map_from_syntax_node(
speculative_args,
// we don't leak these spans into any query so its fine to make them absolute
SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
TextSize::new(0),
),
SpanMapRef::RealSpanMap(&RealSpanMap::empty(SpanAnchor::DUMMY.file_id)),
)
}
MacroDefKind::Declarative(it) => {
@ -261,15 +261,13 @@ pub fn expand_speculative(
};
let expand_to = macro_expand_to(db, actual_macro_call);
let (node, mut rev_tmap) =
token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to);
rev_tmap.real_file = false;
let (node, rev_tmap) = token_tree_to_syntax_node(db, &speculative_expansion.value, expand_to);
let syntax_node = node.syntax_node();
let token = rev_tmap
.ranges_with_span(tt::SpanData {
range: token_to_map.text_range(),
anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID },
anchor: SpanAnchor::DUMMY,
ctx: SyntaxContextId::DUMMY,
})
.filter_map(|range| syntax_node.covering_element(range).into_token())
@ -310,7 +308,7 @@ fn parse_or_expand_with_err(
fn parse_macro_expansion(
db: &dyn ExpandDatabase,
macro_file: MacroFile,
) -> ExpandResult<(Parse<SyntaxNode>, Arc<SpanMap>)> {
) -> ExpandResult<(Parse<SyntaxNode>, Arc<ExpansionSpanMap>)> {
let _p = profile::span("parse_macro_expansion");
let mbe::ValueResult { value: tt, err } = db.macro_expand(macro_file.macro_call_id);
@ -319,8 +317,7 @@ fn parse_macro_expansion(
tracing::debug!("expanded = {}", tt.as_debug_string());
tracing::debug!("kind = {:?}", expand_to);
let (parse, mut rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to);
rev_token_map.real_file = false;
let (parse, rev_token_map) = token_tree_to_syntax_node(db, &tt, expand_to);
ExpandResult { value: (parse, Arc::new(rev_token_map)), err }
}
@ -366,18 +363,21 @@ fn macro_arg(
{
ValueResult::ok(Some(Arc::new(arg.0.clone())))
} else {
//FIXME: clean this up, the ast id map lookup is done twice here
let (parse, map) = match loc.kind.file_id().repr() {
HirFileIdRepr::FileId(file_id) => {
(db.parse(file_id).to_syntax(), Arc::new(Default::default()))
let syntax = db.parse(file_id).to_syntax();
(syntax, SpanMap::RealSpanMap(db.real_span_map(file_id)))
}
HirFileIdRepr::MacroFile(macro_file) => {
let (parse, map) = db.parse_macro_expansion(macro_file).value;
(parse, map)
(parse, SpanMap::ExpansionSpanMap(map))
}
};
let root = parse.syntax_node();
let (syntax, offset, ast_id) = match loc.kind {
let syntax = match loc.kind {
MacroCallKind::FnLike { ast_id, .. } => {
let node = &ast_id.to_ptr(db).to_node(&root);
let offset = node.syntax().text_range().start();
@ -386,7 +386,7 @@ fn macro_arg(
if let Some(e) = mismatched_delimiters(&tt) {
return ValueResult::only_err(e);
}
(tt, offset, ast_id.value.erase())
tt
}
None => {
return ValueResult::only_err(Arc::new(Box::new([
@ -396,15 +396,9 @@ fn macro_arg(
}
}
MacroCallKind::Derive { ast_id, .. } => {
let syntax_node = ast_id.to_ptr(db).to_node(&root).syntax().clone();
let offset = syntax_node.text_range().start();
(syntax_node, offset, ast_id.value.erase())
}
MacroCallKind::Attr { ast_id, .. } => {
let syntax_node = ast_id.to_ptr(db).to_node(&root).syntax().clone();
let offset = syntax_node.text_range().start();
(syntax_node, offset, ast_id.value.erase())
ast_id.to_ptr(db).to_node(&root).syntax().clone()
}
MacroCallKind::Attr { ast_id, .. } => ast_id.to_ptr(db).to_node(&root).syntax().clone(),
};
let censor = censor_for_macro_input(&loc, &syntax);
// let mut fixups = fixup::fixup_syntax(&node);
@ -416,13 +410,8 @@ fn macro_arg(
// fixups.replace,
// fixups.append,
// );
let mut tt = mbe::syntax_node_to_token_tree_censored(
&syntax,
SpanAnchor { file_id: loc.kind.file_id(), ast_id },
offset,
&map,
censor,
);
let mut tt = mbe::syntax_node_to_token_tree_censored(&syntax, map.as_ref(), censor);
if loc.def.is_proc_macro() {
// proc macros expect their inputs without parentheses, MBEs expect it with them included
@ -492,18 +481,19 @@ fn decl_macro_expander(
let is_2021 = db.crate_graph()[def_crate].edition >= Edition::Edition2021;
let (root, map) = match id.file_id.repr() {
HirFileIdRepr::FileId(file_id) => {
(db.parse(file_id).syntax_node(), Arc::new(Default::default()))
// FIXME: Arc
// FIXME: id.to_ptr duplicated, expensive
(db.parse(file_id).syntax_node(), SpanMap::RealSpanMap(db.real_span_map(file_id)))
}
HirFileIdRepr::MacroFile(macro_file) => {
let (parse, map) = db.parse_macro_expansion(macro_file).value;
(parse.syntax_node(), map)
(parse.syntax_node(), SpanMap::ExpansionSpanMap(map))
}
};
let transparency = |node| {
// ... would be nice to have the item tree here
let attrs =
RawAttrs::new(db, SpanAnchor::DUMMY, node, &Default::default()).filter(db, def_crate);
let attrs = RawAttrs::new(db, node, map.as_ref()).filter(db, def_crate);
match &*attrs
.iter()
.find(|it| {
@ -526,12 +516,7 @@ fn decl_macro_expander(
ast::Macro::MacroRules(macro_rules) => (
match macro_rules.token_tree() {
Some(arg) => {
let tt = mbe::syntax_node_to_token_tree(
arg.syntax(),
SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() },
macro_rules.syntax().text_range().start(),
&map,
);
let tt = mbe::syntax_node_to_token_tree(arg.syntax(), map.as_ref());
let mac = mbe::DeclarativeMacro::parse_macro_rules(&tt, is_2021);
mac
}
@ -545,12 +530,7 @@ fn decl_macro_expander(
ast::Macro::MacroDef(macro_def) => (
match macro_def.body() {
Some(arg) => {
let tt = mbe::syntax_node_to_token_tree(
arg.syntax(),
SpanAnchor { file_id: id.file_id, ast_id: id.value.erase() },
macro_def.syntax().text_range().start(),
&map,
);
let tt = mbe::syntax_node_to_token_tree(arg.syntax(), map.as_ref());
let mac = mbe::DeclarativeMacro::parse_macro2(&tt, is_2021);
mac
}
@ -591,10 +571,16 @@ fn macro_expand(
// FIXME: add firewall query for this?
let hir_file_id = loc.kind.file_id();
let (root, map) = match hir_file_id.repr() {
HirFileIdRepr::FileId(file_id) => (db.parse(file_id).syntax_node(), None),
HirFileIdRepr::FileId(file_id) => {
// FIXME: query for span map
(
db.parse(file_id).syntax_node(),
SpanMap::RealSpanMap(db.real_span_map(file_id)),
)
}
HirFileIdRepr::MacroFile(macro_file) => {
let (parse, map) = db.parse_macro_expansion(macro_file).value;
(parse.syntax_node(), Some(map))
(parse.syntax_node(), SpanMap::ExpansionSpanMap(map))
}
};
let MacroCallKind::Derive { ast_id, .. } = loc.kind else { unreachable!() };
@ -602,23 +588,7 @@ fn macro_expand(
// FIXME: we might need to remove the spans from the input to the derive macro here
let _censor = censor_for_macro_input(&loc, node.syntax());
let _t;
expander.expand(
db,
macro_call_id,
&node,
match &map {
Some(map) => map,
None => {
_t = map_from_syntax_node(
node.syntax(),
SpanAnchor { file_id: hir_file_id, ast_id: ast_id.value.erase() },
node.syntax().text_range().start(),
);
&_t
}
},
)
expander.expand(db, macro_call_id, &node, map.as_ref())
}
_ => {
let ValueResult { value, err } = db.macro_arg(macro_call_id);
@ -732,7 +702,7 @@ fn token_tree_to_syntax_node(
db: &dyn ExpandDatabase,
tt: &tt::Subtree,
expand_to: ExpandTo,
) -> (Parse<SyntaxNode>, SpanMap) {
) -> (Parse<SyntaxNode>, ExpansionSpanMap) {
let entry_point = match expand_to {
ExpandTo::Statements => mbe::TopEntryPoint::MacroStmts,
ExpandTo::Items => mbe::TopEntryPoint::MacroItems,
@ -741,14 +711,14 @@ fn token_tree_to_syntax_node(
ExpandTo::Expr => mbe::TopEntryPoint::Expr,
};
let mut tm = mbe::token_tree_to_syntax_node(tt, entry_point);
// now what the hell is going on here
// FIXME: now what the hell is going on here
tm.1.span_map.sort_by(|(_, a), (_, b)| {
a.anchor.file_id.cmp(&b.anchor.file_id).then_with(|| {
let map = db.ast_id_map(a.anchor.file_id);
map.get_raw(a.anchor.ast_id)
let map = db.ast_id_map(a.anchor.file_id.into());
map.get_erased(a.anchor.ast_id)
.text_range()
.start()
.cmp(&map.get_raw(b.anchor.ast_id).text_range().start())
.cmp(&map.get_erased(b.anchor.ast_id).text_range().start())
})
});
tm

78
crates/hir-expand/src/eager.rs
View file

@ -19,7 +19,7 @@
//!
//! See the full discussion : <https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Eager.20expansion.20of.20built-in.20macros>
use base_db::{
span::{SpanAnchor, SyntaxContextId, ROOT_ERASED_FILE_AST_ID},
span::{SpanAnchor, SyntaxContextId},
CrateId,
};
use rustc_hash::FxHashMap;
@ -30,8 +30,9 @@ use crate::{
ast::{self, AstNode},
db::ExpandDatabase,
mod_path::ModPath,
EagerCallInfo, ExpandError, ExpandResult, ExpandTo, InFile, MacroCallId, MacroCallKind,
MacroCallLoc, MacroDefId, MacroDefKind, SpanMap,
span::{RealSpanMap, SpanMapRef},
EagerCallInfo, ExpandError, ExpandResult, ExpandTo, ExpansionSpanMap, InFile, MacroCallId,
MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind,
};
pub fn expand_eager_macro_input(
@ -39,6 +40,7 @@ pub fn expand_eager_macro_input(
krate: CrateId,
macro_call: InFile<ast::MacroCall>,
def: MacroDefId,
call_site: SyntaxContextId,
resolver: &dyn Fn(ModPath) -> Option<MacroDefId>,
) -> ExpandResult<Option<MacroCallId>> {
let ast_map = db.ast_id_map(macro_call.file_id);
@ -55,18 +57,10 @@ pub fn expand_eager_macro_input(
krate,
eager: None,
kind: MacroCallKind::FnLike { ast_id: call_id, expand_to: ExpandTo::Expr },
// FIXME
call_site: SyntaxContextId::ROOT,
call_site,
});
let ExpandResult { value: (arg_exp, arg_exp_map), err: parse_err } =
db.parse_macro_expansion(arg_id.as_macro_file());
// we need this map here as the expansion of the eager input fake file loses whitespace ...
// let mut ws_mapping = FxHashMap::default();
// if let Some((tm)) = db.macro_arg(arg_id).value.as_deref() {
// ws_mapping.extend(tm.entries().filter_map(|(id, range)| {
// Some((arg_exp_map.first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)?, range))
// }));
// }
let ExpandResult { value: expanded_eager_input, err } = {
eager_macro_recur(
@ -74,6 +68,7 @@ pub fn expand_eager_macro_input(
&arg_exp_map,
InFile::new(arg_id.as_file(), arg_exp.syntax_node()),
krate,
call_site,
resolver,
)
};
@ -83,44 +78,12 @@ pub fn expand_eager_macro_input(
return ExpandResult { value: None, err };
};
// FIXME: Spans!
let mut subtree = mbe::syntax_node_to_token_tree(
&expanded_eager_input,
// is this right?
SpanAnchor { file_id: arg_id.as_file(), ast_id: ROOT_ERASED_FILE_AST_ID },
TextSize::new(0),
// FIXME: Spans! `eager_macro_recur` needs to fill out a span map for us
&Default::default(),
RealSpanMap::empty(<SpanAnchor as tt::SpanAnchor>::DUMMY.file_id),
);
// let og_tmap = if let Some(tt) = macro_call.value.token_tree() {
// let mut ids_used = FxHashSet::default();
// let mut og_tmap = mbe::syntax_node_to_token_map(tt.syntax());
// // The tokenmap and ids of subtree point into the expanded syntax node, but that is inaccessible from the outside
// // so we need to remap them to the original input of the eager macro.
// subtree.visit_ids(&mut |id| {
// // Note: we discard all token ids of braces and the like here, but that's not too bad and only a temporary fix
// if let Some(range) = expanded_eager_input_token_map
// .first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)
// {
// // remap from expanded eager input to eager input expansion
// if let Some(og_range) = mapping.get(&range) {
// // remap from eager input expansion to original eager input
// if let Some(&og_range) = ws_mapping.get(og_range) {
// if let Some(og_token) = og_tmap.token_by_range(og_range) {
// ids_used.insert(og_token);
// return og_token;
// }
// }
// }
// }
// tt::TokenId::UNSPECIFIED
// });
// og_tmap.filter(|id| ids_used.contains(&id));
// og_tmap
// } else {
// Default::default()
// };
subtree.delimiter = crate::tt::Delimiter::UNSPECIFIED;
let loc = MacroCallLoc {
@ -132,8 +95,7 @@ pub fn expand_eager_macro_input(
error: err.clone(),
})),
kind: MacroCallKind::FnLike { ast_id: call_id, expand_to },
// FIXME
call_site: SyntaxContextId::ROOT,
call_site,
};
ExpandResult { value: Some(db.intern_macro_call(loc)), err }
@ -144,7 +106,8 @@ fn lazy_expand(
def: &MacroDefId,
macro_call: InFile<ast::MacroCall>,
krate: CrateId,
) -> ExpandResult<(InFile<Parse<SyntaxNode>>, Arc<SpanMap>)> {
call_site: SyntaxContextId,
) -> ExpandResult<(InFile<Parse<SyntaxNode>>, Arc<ExpansionSpanMap>)> {
let ast_id = db.ast_id_map(macro_call.file_id).ast_id(&macro_call.value);
let expand_to = ExpandTo::from_call_site(&macro_call.value);
@ -153,8 +116,8 @@ fn lazy_expand(
db,
krate,
MacroCallKind::FnLike { ast_id, expand_to },
// FIXME
SyntaxContextId::ROOT,
// FIXME: This is wrong
call_site,
);
let macro_file = id.as_macro_file();
@ -164,9 +127,10 @@ fn lazy_expand(
fn eager_macro_recur(
db: &dyn ExpandDatabase,
hygiene: &SpanMap,
hygiene: &ExpansionSpanMap,
curr: InFile<SyntaxNode>,
krate: CrateId,
call_site: SyntaxContextId,
macro_resolver: &dyn Fn(ModPath) -> Option<MacroDefId>,
) -> ExpandResult<Option<(SyntaxNode, FxHashMap<TextRange, TextRange>)>> {
let original = curr.value.clone_for_update();
@ -204,7 +168,10 @@ fn eager_macro_recur(
continue;
}
};
let def = match call.path().and_then(|path| ModPath::from_src(db, path, hygiene)) {
let def = match call
.path()
.and_then(|path| ModPath::from_src(db, path, SpanMapRef::ExpansionSpanMap(hygiene)))
{
Some(path) => match macro_resolver(path.clone()) {
Some(def) => def,
None => {
@ -225,6 +192,8 @@ fn eager_macro_recur(
krate,
curr.with_value(call.clone()),
def,
// FIXME: This call site is not quite right I think? We probably need to mark it?
call_site,
macro_resolver,
);
match value {
@ -260,7 +229,7 @@ fn eager_macro_recur(
| MacroDefKind::BuiltInDerive(..)
| MacroDefKind::ProcMacro(..) => {
let ExpandResult { value: (parse, tm), err } =
lazy_expand(db, &def, curr.with_value(call.clone()), krate);
lazy_expand(db, &def, curr.with_value(call.clone()), krate, call_site);
// replace macro inside
let ExpandResult { value, err: error } = eager_macro_recur(
@ -269,6 +238,7 @@ fn eager_macro_recur(
// FIXME: We discard parse errors here
parse.as_ref().map(|it| it.syntax_node()),
krate,
call_site,
macro_resolver,
);
let err = err.or(error);

293
crates/hir-expand/src/files.rs Normal file
View file

@ -0,0 +1,293 @@
use std::iter;
use base_db::{
span::{HirFileId, HirFileIdRepr, MacroFile, SyntaxContextId},
FileRange,
};
use either::Either;
use syntax::{AstNode, SyntaxNode, SyntaxToken, TextRange};
use crate::{db, ExpansionInfo, HirFileIdExt as _};
// FIXME: Make an InRealFile wrapper
/// `InFile<T>` stores a value of `T` inside a particular file/syntax tree.
///
/// Typical usages are:
///
/// * `InFile<SyntaxNode>` -- syntax node in a file
/// * `InFile<ast::FnDef>` -- ast node in a file
/// * `InFile<TextSize>` -- offset in a file
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
pub struct InFile<T> {
pub file_id: HirFileId,
pub value: T,
}
impl<T> InFile<T> {
pub fn new(file_id: HirFileId, value: T) -> InFile<T> {
InFile { file_id, value }
}
pub fn with_value<U>(&self, value: U) -> InFile<U> {
InFile::new(self.file_id, value)
}
pub fn map<F: FnOnce(T) -> U, U>(self, f: F) -> InFile<U> {
InFile::new(self.file_id, f(self.value))
}
pub fn as_ref(&self) -> InFile<&T> {
self.with_value(&self.value)
}
pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode {
db.parse_or_expand(self.file_id)
}
}
impl<T: Clone> InFile<&T> {
pub fn cloned(&self) -> InFile<T> {
self.with_value(self.value.clone())
}
}
impl<T> InFile<Option<T>> {
pub fn transpose(self) -> Option<InFile<T>> {
let value = self.value?;
Some(InFile::new(self.file_id, value))
}
}
impl<L, R> InFile<Either<L, R>> {
pub fn transpose(self) -> Either<InFile<L>, InFile<R>> {
match self.value {
Either::Left(l) => Either::Left(InFile::new(self.file_id, l)),
Either::Right(r) => Either::Right(InFile::new(self.file_id, r)),
}
}
}
impl InFile<&SyntaxNode> {
pub fn ancestors_with_macros(
self,
db: &dyn db::ExpandDatabase,
) -> impl Iterator<Item = InFile<SyntaxNode>> + Clone + '_ {
iter::successors(Some(self.cloned()), move |node| match node.value.parent() {
Some(parent) => Some(node.with_value(parent)),
None => node.file_id.call_node(db),
})
}
/// Skips the attributed item that caused the macro invocation we are climbing up
pub fn ancestors_with_macros_skip_attr_item(
self,
db: &dyn db::ExpandDatabase,
) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
let succ = move |node: &InFile<SyntaxNode>| match node.value.parent() {
Some(parent) => Some(node.with_value(parent)),
None => {
let parent_node = node.file_id.call_node(db)?;
if node.file_id.is_attr_macro(db) {
// macro call was an attributed item, skip it
// FIXME: does this fail if this is a direct expansion of another macro?
parent_node.map(|node| node.parent()).transpose()
} else {
Some(parent_node)
}
}
};
iter::successors(succ(&self.cloned()), succ)
}
/// Falls back to the macro call range if the node cannot be mapped up fully.
///
/// For attributes and derives, this will point back to the attribute only.
/// For the entire item use [`InFile::original_file_range_full`].
pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() },
HirFileIdRepr::MacroFile(mac_file) => {
if let Some((res, ctxt)) =
ExpansionInfo::new(db, mac_file).map_node_range_up(db, self.value.text_range())
{
// FIXME: Figure out an API that makes proper use of ctx, this only exists to
// keep pre-token map rewrite behaviour.
if ctxt.is_root() {
return res;
}
}
// Fall back to whole macro call.
let loc = db.lookup_intern_macro_call(mac_file.macro_call_id);
loc.kind.original_call_range(db)
}
}
}
/// Falls back to the macro call range if the node cannot be mapped up fully.
pub fn original_file_range_full(self, db: &dyn db::ExpandDatabase) -> FileRange {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() },
HirFileIdRepr::MacroFile(mac_file) => {
if let Some((res, ctxt)) =
ExpansionInfo::new(db, mac_file).map_node_range_up(db, self.value.text_range())
{
// FIXME: Figure out an API that makes proper use of ctx, this only exists to
// keep pre-token map rewrite behaviour.
if ctxt.is_root() {
return res;
}
}
// Fall back to whole macro call.
let loc = db.lookup_intern_macro_call(mac_file.macro_call_id);
loc.kind.original_call_range_with_body(db)
}
}
}
/// Attempts to map the syntax node back up its macro calls.
pub fn original_file_range_opt(
self,
db: &dyn db::ExpandDatabase,
) -> Option<(FileRange, SyntaxContextId)> {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => {
Some((FileRange { file_id, range: self.value.text_range() }, SyntaxContextId::ROOT))
}
HirFileIdRepr::MacroFile(mac_file) => {
ExpansionInfo::new(db, mac_file).map_node_range_up(db, self.value.text_range())
}
}
}
pub fn original_syntax_node(self, db: &dyn db::ExpandDatabase) -> Option<InFile<SyntaxNode>> {
// This kind of upmapping can only be achieved in attribute expanded files,
// as we don't have node inputs otherwise and therefore can't find an `N` node in the input
let Some(file_id) = self.file_id.macro_file() else {
return Some(self.map(Clone::clone));
};
if !self.file_id.is_attr_macro(db) {
return None;
}
let (FileRange { file_id, range }, ctx) =
ExpansionInfo::new(db, file_id).map_node_range_up(db, self.value.text_range())?;
// FIXME: Figure out an API that makes proper use of ctx, this only exists to
// keep pre-token map rewrite behaviour.
if !ctx.is_root() {
return None;
}
let anc = db.parse(file_id).syntax_node().covering_element(range);
let kind = self.value.kind();
// FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes?
let value = anc.ancestors().find(|it| it.kind() == kind)?;
Some(InFile::new(file_id.into(), value))
}
}
impl InFile<SyntaxToken> {
pub fn upmap_once(
self,
db: &dyn db::ExpandDatabase,
) -> Option<InFile<smallvec::SmallVec<[TextRange; 1]>>> {
Some(self.file_id.expansion_info(db)?.map_range_up_once(db, self.value.text_range()))
}
/// Falls back to the macro call range if the node cannot be mapped up fully.
pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() },
HirFileIdRepr::MacroFile(mac_file) => {
if let Some(res) = self.original_file_range_opt(db) {
return res;
}
// Fall back to whole macro call.
let loc = db.lookup_intern_macro_call(mac_file.macro_call_id);
loc.kind.original_call_range(db)
}
}
}
/// Attempts to map the syntax node back up its macro calls.
pub fn original_file_range_opt(self, db: &dyn db::ExpandDatabase) -> Option<FileRange> {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => {
Some(FileRange { file_id, range: self.value.text_range() })
}
HirFileIdRepr::MacroFile(_) => {
let (range, ctxt) = ascend_range_up_macros(db, self.map(|it| it.text_range()));
// FIXME: Figure out an API that makes proper use of ctx, this only exists to
// keep pre-token map rewrite behaviour.
if ctxt.is_root() {
Some(range)
} else {
None
}
}
}
}
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
pub struct InMacroFile<T> {
pub file_id: MacroFile,
pub value: T,
}
impl<T> From<InMacroFile<T>> for InFile<T> {
fn from(macro_file: InMacroFile<T>) -> Self {
InFile { file_id: macro_file.file_id.into(), value: macro_file.value }
}
}
pub fn ascend_range_up_macros(
db: &dyn db::ExpandDatabase,
range: InFile<TextRange>,
) -> (FileRange, SyntaxContextId) {
match range.file_id.repr() {
HirFileIdRepr::FileId(file_id) => {
(FileRange { file_id, range: range.value }, SyntaxContextId::ROOT)
}
HirFileIdRepr::MacroFile(m) => {
ExpansionInfo::new(db, m).map_token_range_up(db, range.value)
}
}
}
impl<N: AstNode> InFile<N> {
pub fn descendants<T: AstNode>(self) -> impl Iterator<Item = InFile<T>> {
self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n))
}
// FIXME: this should return `Option<InFileNotHirFile<N>>`
pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option<InFile<N>> {
// This kind of upmapping can only be achieved in attribute expanded files,
// as we don't have node inputs otherwise and therefore can't find an `N` node in the input
let Some(file_id) = self.file_id.macro_file() else {
return Some(self);
};
if !self.file_id.is_attr_macro(db) {
return None;
}
let (FileRange { file_id, range }, ctx) = ExpansionInfo::new(db, file_id)
.map_node_range_up(db, self.value.syntax().text_range())?;
// FIXME: Figure out an API that makes proper use of ctx, this only exists to
// keep pre-token map rewrite behaviour.
if !ctx.is_root() {
return None;
}
// FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes?
let anc = db.parse(file_id).syntax_node().covering_element(range);
let value = anc.ancestors().find_map(N::cast)?;
return Some(InFile::new(file_id.into(), value));
}
pub fn syntax(&self) -> InFile<&SyntaxNode> {
self.with_value(self.value.syntax())
}
}

67
crates/hir-expand/src/hygiene.rs
View file

@ -2,7 +2,9 @@
//!
//! Specifically, `ast` + `Hygiene` allows you to create a `Name`. Note that, at
//! this moment, this is horribly incomplete and handles only `$crate`.
use base_db::span::{MacroCallId, SyntaxContextId};
use std::iter;
use base_db::span::{MacroCallId, SpanData, SyntaxContextId};
use crate::db::ExpandDatabase;
@ -48,6 +50,39 @@ pub enum Transparency {
Opaque,
}
pub fn span_with_def_site_ctxt(
db: &dyn ExpandDatabase,
span: SpanData,
expn_id: MacroCallId,
) -> SpanData {
span_with_ctxt_from_mark(db, span, expn_id, Transparency::Opaque)
}
pub fn span_with_call_site_ctxt(
db: &dyn ExpandDatabase,
span: SpanData,
expn_id: MacroCallId,
) -> SpanData {
span_with_ctxt_from_mark(db, span, expn_id, Transparency::Transparent)
}
pub fn span_with_mixed_site_ctxt(
db: &dyn ExpandDatabase,
span: SpanData,
expn_id: MacroCallId,
) -> SpanData {
span_with_ctxt_from_mark(db, span, expn_id, Transparency::SemiTransparent)
}
fn span_with_ctxt_from_mark(
db: &dyn ExpandDatabase,
span: SpanData,
expn_id: MacroCallId,
transparency: Transparency,
) -> SpanData {
SpanData { ctx: db.apply_mark(SyntaxContextId::ROOT, expn_id, transparency), ..span }
}
pub(super) fn apply_mark(
db: &dyn ExpandDatabase,
ctxt: SyntaxContextId,
@ -65,7 +100,7 @@ pub(super) fn apply_mark(
call_site_ctxt.normalize_to_macro_rules(db)
};
if call_site_ctxt.is_root(db) {
if call_site_ctxt.is_root() {
return apply_mark_internal(db, ctxt, Some(call_id), transparency);
}
@ -131,7 +166,6 @@ fn apply_mark_internal(
})
}
pub trait SyntaxContextExt {
fn is_root(self, db: &dyn ExpandDatabase) -> bool;
fn normalize_to_macro_rules(self, db: &dyn ExpandDatabase) -> Self;
fn normalize_to_macros_2_0(self, db: &dyn ExpandDatabase) -> Self;
fn parent_ctxt(self, db: &dyn ExpandDatabase) -> Self;
@ -148,9 +182,6 @@ fn handle_self_ref(p: SyntaxContextId, n: SyntaxContextId) -> SyntaxContextId {
}
impl SyntaxContextExt for SyntaxContextId {
fn is_root(self, db: &dyn ExpandDatabase) -> bool {
db.lookup_intern_syntax_context(self).outer_expn.is_none()
}
fn normalize_to_macro_rules(self, db: &dyn ExpandDatabase) -> Self {
handle_self_ref(self, db.lookup_intern_syntax_context(self).opaque_and_semitransparent)
}
@ -164,20 +195,20 @@ impl SyntaxContextExt for SyntaxContextId {
let data = db.lookup_intern_syntax_context(self);
(data.outer_expn, data.outer_transparency)
}
fn marks(mut self, db: &dyn ExpandDatabase) -> Vec<(Option<MacroCallId>, Transparency)> {
let mut marks = Vec::new();
while self != SyntaxContextId::ROOT {
marks.push(self.outer_mark(db));
self = self.parent_ctxt(db);
}
fn marks(self, db: &dyn ExpandDatabase) -> Vec<(Option<MacroCallId>, Transparency)> {
let mut marks = marks_rev(self, db).collect::<Vec<_>>();
marks.reverse();
marks
}
}
// pub(super) fn with_ctxt_from_mark(db: &ExpandDatabase, file_id: HirFileId) {
// self.with_ctxt_from_mark(expn_id, Transparency::Transparent)
// }
// pub(super) fn with_call_site_ctxt(db: &ExpandDatabase, file_id: HirFileId) {
// self.with_ctxt_from_mark(expn_id, Transparency::Transparent)
// }
// FIXME: Make this a SyntaxContextExt method once we have RPIT
pub fn marks_rev(
ctxt: SyntaxContextId,
db: &dyn ExpandDatabase,
) -> impl Iterator<Item = (Option<MacroCallId>, Transparency)> + '_ {
iter::successors(Some(ctxt), move |&mark| {
Some(mark.parent_ctxt(db)).filter(|&it| it != SyntaxContextId::ROOT)
})
.map(|ctx| ctx.outer_mark(db))
}

551
crates/hir-expand/src/lib.rs
View file

@ -18,25 +18,25 @@ pub mod quote;
pub mod eager;
pub mod mod_path;
pub mod attrs;
pub mod span;
pub mod files;
// mod fixup;
use triomphe::Arc;
use std::{fmt, hash::Hash, iter};
use std::{fmt, hash::Hash};
use base_db::{
span::{HirFileIdRepr, SyntaxContextId},
span::{HirFileIdRepr, SpanData, SyntaxContextId},
CrateId, FileId, FileRange, ProcMacroKind,
};
use either::Either;
use syntax::{
algo::{self, skip_trivia_token},
ast::{self, AstNode, HasDocComments},
AstPtr, Direction, SyntaxNode, SyntaxNodePtr, SyntaxToken, TextSize,
SyntaxNode, SyntaxToken, TextRange, TextSize,
};
use crate::{
ast_id_map::{AstIdNode, ErasedFileAstId, FileAstId},
attrs::AttrId,
builtin_attr_macro::BuiltinAttrExpander,
builtin_derive_macro::BuiltinDeriveExpander,
@ -44,12 +44,15 @@ use crate::{
db::TokenExpander,
mod_path::ModPath,
proc_macro::ProcMacroExpander,
span::ExpansionSpanMap,
};
pub use crate::ast_id_map::{AstId, ErasedAstId, ErasedFileAstId};
pub use crate::files::{InFile, InMacroFile};
pub use base_db::span::{HirFileId, MacroCallId, MacroFile};
pub use mbe::ValueResult;
pub type SpanMap = ::mbe::TokenMap<tt::SpanData>;
pub type DeclarativeMacro = ::mbe::DeclarativeMacro<tt::SpanData>;
pub mod tt {
@ -103,7 +106,7 @@ impl fmt::Display for ExpandError {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct MacroCallLoc {
pub def: MacroDefId,
pub(crate) krate: CrateId,
pub krate: CrateId,
/// Some if this is a macro call for an eager macro. Note that this is `None`
/// for the eager input macro file.
eager: Option<Box<EagerCallInfo>>,
@ -247,8 +250,7 @@ impl HirFileIdExt for HirFileId {
/// Return expansion information if it is a macro-expansion file
fn expansion_info(self, db: &dyn db::ExpandDatabase) -> Option<ExpansionInfo> {
let macro_file = self.macro_file()?;
ExpansionInfo::new(db, macro_file)
Some(ExpansionInfo::new(db, self.macro_file()?))
}
fn as_builtin_derive_attr_node(
@ -340,15 +342,14 @@ impl MacroDefId {
}
pub fn ast_id(&self) -> Either<AstId<ast::Macro>, AstId<ast::Fn>> {
let id = match self.kind {
match self.kind {
MacroDefKind::ProcMacro(.., id) => return Either::Right(id),
MacroDefKind::Declarative(id)
| MacroDefKind::BuiltIn(_, id)
| MacroDefKind::BuiltInAttr(_, id)
| MacroDefKind::BuiltInDerive(_, id)
| MacroDefKind::BuiltInEager(_, id) => id,
};
Either::Left(id)
| MacroDefKind::BuiltInEager(_, id) => Either::Left(id),
}
}
pub fn is_proc_macro(&self) -> bool {
@ -390,6 +391,18 @@ impl MacroDefId {
}
impl MacroCallLoc {
pub fn span(&self, db: &dyn db::ExpandDatabase) -> SpanData {
let ast_id = self.kind.erased_ast_id();
let file_id = self.kind.file_id();
let range = db.ast_id_map(file_id).get_erased(ast_id).text_range();
match file_id.repr() {
HirFileIdRepr::FileId(file_id) => db.real_span_map(file_id).span_for_range(range),
HirFileIdRepr::MacroFile(m) => {
db.parse_macro_expansion(m).value.1.span_for_range(range)
}
}
}
pub fn to_node(&self, db: &dyn db::ExpandDatabase) -> InFile<SyntaxNode> {
match self.kind {
MacroCallKind::FnLike { ast_id, .. } => {
@ -430,17 +443,15 @@ impl MacroCallLoc {
match self.kind {
MacroCallKind::FnLike { expand_to, .. } => expand_to,
MacroCallKind::Derive { .. } => ExpandTo::Items,
MacroCallKind::Attr { .. } if self.def.is_attribute_derive() => ExpandTo::Statements,
MacroCallKind::Attr { .. } if self.def.is_attribute_derive() => ExpandTo::Items,
MacroCallKind::Attr { .. } => {
// is this always correct?
// FIXME(stmt_expr_attributes)
ExpandTo::Items
}
}
}
}
// FIXME: attribute indices do not account for nested `cfg_attr`
impl MacroCallKind {
/// Returns the file containing the macro invocation.
fn file_id(&self) -> HirFileId {
@ -451,6 +462,14 @@ impl MacroCallKind {
}
}
fn erased_ast_id(&self) -> ErasedFileAstId {
match *self {
MacroCallKind::FnLike { ast_id: InFile { value, .. }, .. } => value.erase(),
MacroCallKind::Derive { ast_id: InFile { value, .. }, .. } => value.erase(),
MacroCallKind::Attr { ast_id: InFile { value, .. }, .. } => value.erase(),
}
}
/// Returns the original file range that best describes the location of this macro call.
///
/// Unlike `MacroCallKind::original_call_range`, this also spans the item of attributes and derives.
@ -518,34 +537,40 @@ impl MacroCallKind {
FileRange { range, file_id }
}
fn arg(&self, db: &dyn db::ExpandDatabase) -> Option<InFile<SyntaxNode>> {
// FIXME: -> InFile<SyntaxNode> it should be impossible for the token tree to be missing at
// this point!
fn arg(&self, db: &dyn db::ExpandDatabase) -> InFile<Option<SyntaxNode>> {
match self {
MacroCallKind::FnLike { ast_id, .. } => ast_id
.to_in_file_node(db)
.map(|it| Some(it.token_tree()?.syntax().clone()))
.transpose(),
MacroCallKind::FnLike { ast_id, .. } => {
ast_id.to_in_file_node(db).map(|it| Some(it.token_tree()?.syntax().clone()))
}
MacroCallKind::Derive { ast_id, .. } => {
Some(ast_id.to_in_file_node(db).syntax().cloned())
ast_id.to_in_file_node(db).syntax().cloned().map(Some)
}
MacroCallKind::Attr { ast_id, .. } => {
Some(ast_id.to_in_file_node(db).syntax().cloned())
ast_id.to_in_file_node(db).syntax().cloned().map(Some)
}
}
}
}
/// ExpansionInfo mainly describes how to map text range between src and expanded macro
// FIXME: can be expensive to create, we should check the use sites and maybe replace them with
// simpler function calls if the map is only used once
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ExpansionInfo {
expanded: InMacroFile<SyntaxNode>,
pub expanded: InMacroFile<SyntaxNode>,
/// The argument TokenTree or item for attributes
arg: InFile<SyntaxNode>,
// FIXME: Can this ever be `None`?
arg: InFile<Option<SyntaxNode>>,
/// The `macro_rules!` or attribute input.
attr_input_or_mac_def: Option<InFile<ast::TokenTree>>,
macro_def: TokenExpander,
macro_arg: Arc<tt::Subtree>,
exp_map: Arc<SpanMap>,
exp_map: Arc<ExpansionSpanMap>,
/// [`None`] if the call is in a real file
arg_map: Option<Arc<ExpansionSpanMap>>,
}
impl ExpansionInfo {
@ -554,81 +579,133 @@ impl ExpansionInfo {
}
pub fn call_node(&self) -> Option<InFile<SyntaxNode>> {
Some(self.arg.with_value(self.arg.value.parent()?))
Some(self.arg.with_value(self.arg.value.as_ref()?.parent()?))
}
/// Map a token down from macro input into the macro expansion.
///
/// The inner workings of this function differ slightly depending on the type of macro we are dealing with:
/// - declarative:
/// For declarative macros, we need to accommodate for the macro definition site(which acts as a second unchanging input)
/// , as tokens can mapped in and out of it.
/// To do this we shift all ids in the expansion by the maximum id of the definition site giving us an easy
/// way to map all the tokens.
/// - attribute:
/// Attributes have two different inputs, the input tokentree in the attribute node and the item
/// the attribute is annotating. Similarly as for declarative macros we need to do a shift here
/// as well. Currently this is done by shifting the attribute input by the maximum id of the item.
/// - function-like and derives:
/// Both of these only have one simple call site input so no special handling is required here.
pub fn map_token_down(
&self,
db: &dyn db::ExpandDatabase,
token: InFile<&SyntaxToken>,
/// Maps the passed in file range down into a macro expansion if it is the input to a macro call.
pub fn map_range_down<'a>(
&'a self,
db: &'a dyn db::ExpandDatabase,
FileRange { file_id, range: absolute_range }: FileRange,
// FIXME: use this for range mapping, so that we can resolve inline format args
_relative_token_offset: Option<TextSize>,
) -> Option<impl Iterator<Item = InFile<SyntaxToken>> + '_> {
assert_eq!(token.file_id, self.arg.file_id);
// FIXME: ret ty should be wrapped in InMacroFile
) -> Option<impl Iterator<Item = InFile<SyntaxToken>> + 'a> {
// search for all entries in the span map that have the given span and return the
// corresponding text ranges inside the expansion
// FIXME: Make this proper
let span_map = &self.exp_map.span_map;
let (start, end) = if span_map
.first()
.map_or(false, |(_, span)| span.anchor.file_id == token.file_id)
.map_or(false, |(_, span)| span.anchor.file_id == file_id)
{
(0, span_map.partition_point(|a| a.1.anchor.file_id == token.file_id))
(0, span_map.partition_point(|a| a.1.anchor.file_id == file_id))
} else {
let start = span_map.partition_point(|a| a.1.anchor.file_id != token.file_id);
(
start,
start + span_map[start..].partition_point(|a| a.1.anchor.file_id == token.file_id),
)
let start = span_map.partition_point(|a| a.1.anchor.file_id != file_id);
(start, start + span_map[start..].partition_point(|a| a.1.anchor.file_id == file_id))
};
let token_text_range = token.value.text_range();
let ast_id_map = db.ast_id_map(token.file_id);
let tokens = span_map[start..end]
.iter()
.filter_map(move |(range, span)| {
let offset = ast_id_map.get_raw(span.anchor.ast_id).text_range().start();
// we need to resolve the relative ranges here to make sure that we are in fact
// considering differently anchored spans (this might occur with proc-macros)
let offset = db
.ast_id_map(span.anchor.file_id.into())
.get_erased(span.anchor.ast_id)
.text_range()
.start();
let abs_range = span.range + offset;
token_text_range.eq(&abs_range).then_some(*range)
absolute_range.eq(&abs_range).then_some(*range)
})
.flat_map(move |range| self.expanded.value.covering_element(range).into_token());
Some(tokens.map(move |token| InFile::new(self.expanded.file_id.into(), token)))
}
/// Map a token up out of the expansion it resides in into the arguments of the macro call of the expansion.
pub fn map_token_up(
/// Maps up the text range out of the expansion hierarchy back into the original file its from.
pub fn map_token_range_up(
&self,
db: &dyn db::ExpandDatabase,
token: InFile<&SyntaxToken>,
) -> Option<InFile<SyntaxToken>> {
self.exp_map.span_for_range(token.value.text_range()).and_then(|span| {
let anchor =
db.ast_id_map(span.anchor.file_id).get_raw(span.anchor.ast_id).text_range().start();
InFile::new(
span.anchor.file_id,
db.parse_or_expand(span.anchor.file_id)
.covering_element(span.range + anchor)
.into_token(),
)
.transpose()
})
range: TextRange,
) -> (FileRange, SyntaxContextId) {
debug_assert!(self.expanded.value.text_range().contains_range(range));
let span = self.exp_map.span_for_range(range);
let anchor_offset = db
.ast_id_map(span.anchor.file_id.into())
.get_erased(span.anchor.ast_id)
.text_range()
.start();
(FileRange { file_id: span.anchor.file_id, range: span.range + anchor_offset }, span.ctx)
}
fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFile) -> Option<ExpansionInfo> {
/// Maps up the text range out of the expansion hierarchy back into the original file its from.
pub fn map_node_range_up(
&self,
db: &dyn db::ExpandDatabase,
range: TextRange,
) -> Option<(FileRange, SyntaxContextId)> {
debug_assert!(self.expanded.value.text_range().contains_range(range));
let mut spans = self.exp_map.spans_for_node_range(range);
let SpanData { range, anchor, ctx } = spans.next()?;
let mut start = range.start();
let mut end = range.end();
for span in spans {
if span.anchor != anchor || span.ctx != ctx {
return None;
}
start = start.min(span.range.start());
end = end.max(span.range.end());
}
let anchor_offset =
db.ast_id_map(anchor.file_id.into()).get_erased(anchor.ast_id).text_range().start();
Some((
FileRange {
file_id: anchor.file_id,
range: TextRange::new(start, end) + anchor_offset,
},
ctx,
))
}
/// Maps up the text range out of the expansion into is macro call.
pub fn map_range_up_once(
&self,
db: &dyn db::ExpandDatabase,
token: TextRange,
) -> InFile<smallvec::SmallVec<[TextRange; 1]>> {
debug_assert!(self.expanded.value.text_range().contains_range(token));
let span = self.exp_map.span_for_range(token);
match &self.arg_map {
None => {
let file_id = span.anchor.file_id.into();
let anchor_offset =
db.ast_id_map(file_id).get_erased(span.anchor.ast_id).text_range().start();
InFile { file_id, value: smallvec::smallvec![span.range + anchor_offset] }
}
Some(arg_map) => {
let arg_range = self
.arg
.value
.as_ref()
.map_or_else(|| TextRange::empty(TextSize::from(0)), |it| it.text_range());
InFile::new(
self.arg.file_id,
arg_map
.ranges_with_span(span)
.filter(|range| range.intersect(arg_range).is_some())
.collect(),
)
}
}
}
pub fn new(db: &dyn db::ExpandDatabase, macro_file: MacroFile) -> ExpansionInfo {
let loc: MacroCallLoc = db.lookup_intern_macro_call(macro_file.macro_call_id);
let arg_tt = loc.kind.arg(db)?;
let arg_tt = loc.kind.arg(db);
let arg_map =
arg_tt.file_id.macro_file().map(|file| db.parse_macro_expansion(file).value.1);
let macro_def = db.macro_expander(loc.def);
let (parse, exp_map) = db.parse_macro_expansion(macro_file).value;
@ -662,331 +739,18 @@ impl ExpansionInfo {
_ => None,
});
Some(ExpansionInfo {
ExpansionInfo {
expanded,
arg: arg_tt,
attr_input_or_mac_def,
macro_arg,
macro_def,
exp_map,
})
}
}
/// `AstId` points to an AST node in any file.
///
/// It is stable across reparses, and can be used as salsa key/value.
pub type AstId<N> = InFile<FileAstId<N>>;
impl<N: AstIdNode> AstId<N> {
pub fn to_node(&self, db: &dyn db::ExpandDatabase) -> N {
self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id))
}
pub fn to_in_file_node(&self, db: &dyn db::ExpandDatabase) -> InFile<N> {
InFile::new(self.file_id, self.to_ptr(db).to_node(&db.parse_or_expand(self.file_id)))
}
pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> AstPtr<N> {
db.ast_id_map(self.file_id).get(self.value)
}
}
pub type ErasedAstId = InFile<ErasedFileAstId>;
impl ErasedAstId {
pub fn to_ptr(&self, db: &dyn db::ExpandDatabase) -> SyntaxNodePtr {
db.ast_id_map(self.file_id).get_raw(self.value)
}
}
/// `InFile<T>` stores a value of `T` inside a particular file/syntax tree.
///
/// Typical usages are:
///
/// * `InFile<SyntaxNode>` -- syntax node in a file
/// * `InFile<ast::FnDef>` -- ast node in a file
/// * `InFile<TextSize>` -- offset in a file
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
pub struct InFile<T> {
pub file_id: HirFileId,
pub value: T,
}
impl<T> InFile<T> {
pub fn new(file_id: HirFileId, value: T) -> InFile<T> {
InFile { file_id, value }
}
pub fn with_value<U>(&self, value: U) -> InFile<U> {
InFile::new(self.file_id, value)
}
pub fn map<F: FnOnce(T) -> U, U>(self, f: F) -> InFile<U> {
InFile::new(self.file_id, f(self.value))
}
pub fn as_ref(&self) -> InFile<&T> {
self.with_value(&self.value)
}
pub fn file_syntax(&self, db: &dyn db::ExpandDatabase) -> SyntaxNode {
db.parse_or_expand(self.file_id)
}
}
impl<T: Clone> InFile<&T> {
pub fn cloned(&self) -> InFile<T> {
self.with_value(self.value.clone())
}
}
impl<T> InFile<Option<T>> {
pub fn transpose(self) -> Option<InFile<T>> {
let value = self.value?;
Some(InFile::new(self.file_id, value))
}
}
impl<L, R> InFile<Either<L, R>> {
pub fn transpose(self) -> Either<InFile<L>, InFile<R>> {
match self.value {
Either::Left(l) => Either::Left(InFile::new(self.file_id, l)),
Either::Right(r) => Either::Right(InFile::new(self.file_id, r)),
arg_map,
}
}
}
impl InFile<&SyntaxNode> {
pub fn ancestors_with_macros(
self,
db: &dyn db::ExpandDatabase,
) -> impl Iterator<Item = InFile<SyntaxNode>> + Clone + '_ {
iter::successors(Some(self.cloned()), move |node| match node.value.parent() {
Some(parent) => Some(node.with_value(parent)),
None => node.file_id.call_node(db),
})
}
/// Skips the attributed item that caused the macro invocation we are climbing up
pub fn ancestors_with_macros_skip_attr_item(
self,
db: &dyn db::ExpandDatabase,
) -> impl Iterator<Item = InFile<SyntaxNode>> + '_ {
let succ = move |node: &InFile<SyntaxNode>| match node.value.parent() {
Some(parent) => Some(node.with_value(parent)),
None => {
let parent_node = node.file_id.call_node(db)?;
if node.file_id.is_attr_macro(db) {
// macro call was an attributed item, skip it
// FIXME: does this fail if this is a direct expansion of another macro?
parent_node.map(|node| node.parent()).transpose()
} else {
Some(parent_node)
}
}
};
iter::successors(succ(&self.cloned()), succ)
}
/// Falls back to the macro call range if the node cannot be mapped up fully.
///
/// For attributes and derives, this will point back to the attribute only.
/// For the entire item use [`InFile::original_file_range_full`].
pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() },
HirFileIdRepr::MacroFile(mac_file) => {
if let Some(res) = self.original_file_range_opt(db) {
return res;
}
// Fall back to whole macro call.
let loc = db.lookup_intern_macro_call(mac_file.macro_call_id);
loc.kind.original_call_range(db)
}
}
}
/// Falls back to the macro call range if the node cannot be mapped up fully.
pub fn original_file_range_full(self, db: &dyn db::ExpandDatabase) -> FileRange {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() },
HirFileIdRepr::MacroFile(mac_file) => {
if let Some(res) = self.original_file_range_opt(db) {
return res;
}
// Fall back to whole macro call.
let loc = db.lookup_intern_macro_call(mac_file.macro_call_id);
loc.kind.original_call_range_with_body(db)
}
}
}
/// Attempts to map the syntax node back up its macro calls.
pub fn original_file_range_opt(self, db: &dyn db::ExpandDatabase) -> Option<FileRange> {
match ascend_node_border_tokens(db, self) {
Some(InFile { file_id, value: (first, last) }) => {
let original_file = file_id.original_file(db);
let range = first.text_range().cover(last.text_range());
if file_id != original_file.into() {
tracing::error!("Failed mapping up more for {:?}", range);
return None;
}
Some(FileRange { file_id: original_file, range })
}
_ if !self.file_id.is_macro() => Some(FileRange {
file_id: self.file_id.original_file(db),
range: self.value.text_range(),
}),
_ => None,
}
}
pub fn original_syntax_node(self, db: &dyn db::ExpandDatabase) -> Option<InFile<SyntaxNode>> {
// This kind of upmapping can only be achieved in attribute expanded files,
// as we don't have node inputs otherwise and therefore can't find an `N` node in the input
if !self.file_id.is_macro() {
return Some(self.map(Clone::clone));
} else if !self.file_id.is_attr_macro(db) {
return None;
}
if let Some(InFile { file_id, value: (first, last) }) = ascend_node_border_tokens(db, self)
{
if file_id.is_macro() {
let range = first.text_range().cover(last.text_range());
tracing::error!("Failed mapping out of macro file for {:?}", range);
return None;
}
// FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes
let anc = algo::least_common_ancestor(&first.parent()?, &last.parent()?)?;
let kind = self.value.kind();
let value = anc.ancestors().find(|it| it.kind() == kind)?;
return Some(InFile::new(file_id, value));
}
None
}
}
impl InFile<SyntaxToken> {
pub fn upmap(self, db: &dyn db::ExpandDatabase) -> Option<InFile<SyntaxToken>> {
let expansion = self.file_id.expansion_info(db)?;
expansion.map_token_up(db, self.as_ref())
}
/// Falls back to the macro call range if the node cannot be mapped up fully.
pub fn original_file_range(self, db: &dyn db::ExpandDatabase) -> FileRange {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => FileRange { file_id, range: self.value.text_range() },
HirFileIdRepr::MacroFile(mac_file) => {
if let Some(res) = self.original_file_range_opt(db) {
return res;
}
// Fall back to whole macro call.
let loc = db.lookup_intern_macro_call(mac_file.macro_call_id);
loc.kind.original_call_range(db)
}
}
}
/// Attempts to map the syntax node back up its macro calls.
pub fn original_file_range_opt(self, db: &dyn db::ExpandDatabase) -> Option<FileRange> {
match self.file_id.repr() {
HirFileIdRepr::FileId(file_id) => {
Some(FileRange { file_id, range: self.value.text_range() })
}
HirFileIdRepr::MacroFile(_) => {
let expansion = self.file_id.expansion_info(db)?;
let InFile { file_id, value } = ascend_call_token(db, &expansion, self)?;
let original_file = file_id.original_file(db);
if file_id != original_file.into() {
return None;
}
Some(FileRange { file_id: original_file, range: value.text_range() })
}
}
}
}
#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
pub struct InMacroFile<T> {
pub file_id: MacroFile,
pub value: T,
}
impl<T> From<InMacroFile<T>> for InFile<T> {
fn from(macro_file: InMacroFile<T>) -> Self {
InFile { file_id: macro_file.file_id.into(), value: macro_file.value }
}
}
// FIXME: Get rid of this
fn ascend_node_border_tokens(
db: &dyn db::ExpandDatabase,
InFile { file_id, value: node }: InFile<&SyntaxNode>,
) -> Option<InFile<(SyntaxToken, SyntaxToken)>> {
let expansion = file_id.expansion_info(db)?;
let first_token = |node: &SyntaxNode| skip_trivia_token(node.first_token()?, Direction::Next);
let last_token = |node: &SyntaxNode| skip_trivia_token(node.last_token()?, Direction::Prev);
// FIXME: Once the token map rewrite is done, this shouldnt need to rely on syntax nodes and tokens anymore
let first = first_token(node)?;
let last = last_token(node)?;
let first = ascend_call_token(db, &expansion, InFile::new(file_id, first))?;
let last = ascend_call_token(db, &expansion, InFile::new(file_id, last))?;
(first.file_id == last.file_id).then(|| InFile::new(first.file_id, (first.value, last.value)))
}
fn ascend_call_token(
db: &dyn db::ExpandDatabase,
expansion: &ExpansionInfo,
token: InFile<SyntaxToken>,
) -> Option<InFile<SyntaxToken>> {
let mut mapping = expansion.map_token_up(db, token.as_ref())?;
loop {
match mapping.file_id.expansion_info(db) {
Some(info) => mapping = info.map_token_up(db, mapping.as_ref())?,
None => return Some(mapping),
}
}
}
impl<N: AstNode> InFile<N> {
pub fn descendants<T: AstNode>(self) -> impl Iterator<Item = InFile<T>> {
self.value.syntax().descendants().filter_map(T::cast).map(move |n| self.with_value(n))
}
// FIXME: this should return `Option<InFileNotHirFile<N>>`
pub fn original_ast_node(self, db: &dyn db::ExpandDatabase) -> Option<InFile<N>> {
// This kind of upmapping can only be achieved in attribute expanded files,
// as we don't have node inputs otherwise and therefore can't find an `N` node in the input
if !self.file_id.is_macro() {
return Some(self);
} else if !self.file_id.is_attr_macro(db) {
return None;
}
if let Some(InFile { file_id, value: (first, last) }) =
ascend_node_border_tokens(db, self.syntax())
{
if file_id.is_macro() {
let range = first.text_range().cover(last.text_range());
tracing::error!("Failed mapping out of macro file for {:?}", range);
return None;
}
// FIXME: This heuristic is brittle and with the right macro may select completely unrelated nodes
let anc = algo::least_common_ancestor(&first.parent()?, &last.parent()?)?;
let value = anc.ancestors().find_map(N::cast)?;
return Some(InFile::new(file_id, value));
}
None
}
pub fn syntax(&self) -> InFile<&SyntaxNode> {
self.with_value(self.value.syntax())
}
}
/// In Rust, macros expand token trees to token trees. When we want to turn a
/// token tree into an AST node, we need to figure out what kind of AST node we
/// want: something like `foo` can be a type, an expression, or a pattern.
@ -1051,9 +815,4 @@ impl ExpandTo {
}
}
#[derive(Debug)]
pub struct UnresolvedMacro {
pub path: ModPath,
}
intern::impl_internable!(ModPath, attrs::AttrInput);

56
crates/hir-expand/src/mod_path.rs
View file

@ -7,9 +7,9 @@ use std::{
use crate::{
db::ExpandDatabase,
hygiene::{SyntaxContextExt, Transparency},
hygiene::{marks_rev, SyntaxContextExt, Transparency},
name::{known, AsName, Name},
SpanMap,
span::SpanMapRef,
};
use base_db::{span::SyntaxContextId, CrateId};
use smallvec::SmallVec;
@ -47,7 +47,7 @@ impl ModPath {
pub fn from_src(
db: &dyn ExpandDatabase,
path: ast::Path,
hygiene: &SpanMap,
hygiene: SpanMapRef<'_>,
) -> Option<ModPath> {
convert_path(db, None, path, hygiene)
}
@ -194,7 +194,7 @@ fn convert_path(
db: &dyn ExpandDatabase,
prefix: Option<ModPath>,
path: ast::Path,
hygiene: &SpanMap,
hygiene: SpanMapRef<'_>,
) -> Option<ModPath> {
let prefix = match path.qualifier() {
Some(qual) => Some(convert_path(db, prefix, qual, hygiene)?),
@ -208,14 +208,14 @@ fn convert_path(
if prefix.is_some() {
return None;
}
resolve_crate_root(
db,
hygiene
.span_for_range(name_ref.syntax().text_range())
.map_or(SyntaxContextId::ROOT, |s| s.ctx),
ModPath::from_kind(
resolve_crate_root(
db,
hygiene.span_for_range(name_ref.syntax().text_range()).ctx,
)
.map(PathKind::DollarCrate)
.unwrap_or(PathKind::Crate),
)
.map(PathKind::DollarCrate)
.map(ModPath::from_kind)?
} else {
let mut res = prefix.unwrap_or_else(|| {
ModPath::from_kind(
@ -265,13 +265,12 @@ fn convert_path(
// We follow what it did anyway :)
if mod_path.segments.len() == 1 && mod_path.kind == PathKind::Plain {
if let Some(_macro_call) = path.syntax().parent().and_then(ast::MacroCall::cast) {
let syn_ctx = hygiene
.span_for_range(segment.syntax().text_range())
.map_or(SyntaxContextId::ROOT, |s| s.ctx);
let syn_ctx = hygiene.span_for_range(segment.syntax().text_range()).ctx;
if let Some(macro_call_id) = db.lookup_intern_syntax_context(syn_ctx).outer_expn {
if db.lookup_intern_macro_call(macro_call_id).def.local_inner {
if let Some(crate_root) = resolve_crate_root(db, syn_ctx) {
mod_path.kind = PathKind::DollarCrate(crate_root);
mod_path.kind = match resolve_crate_root(db, syn_ctx) {
Some(crate_root) => PathKind::DollarCrate(crate_root),
None => PathKind::Crate,
}
}
}
@ -289,30 +288,19 @@ pub fn resolve_crate_root(db: &dyn ExpandDatabase, mut ctxt: SyntaxContextId) ->
// definitions actually produced by `macro` and `macro` definitions produced by
// `macro_rules!`, but at least such configurations are not stable yet.
ctxt = ctxt.normalize_to_macro_rules(db);
let mut iter = ctxt.marks(db).into_iter().rev().peekable();
let mut iter = marks_rev(ctxt, db).peekable();
let mut result_mark = None;
// Find the last opaque mark from the end if it exists.
while let Some(&(mark, transparency)) = iter.peek() {
if transparency == Transparency::Opaque {
result_mark = Some(mark);
iter.next();
} else {
break;
}
while let Some(&(mark, Transparency::Opaque)) = iter.peek() {
result_mark = Some(mark);
iter.next();
}
// Then find the last semi-transparent mark from the end if it exists.
for (mark, transparency) in iter {
if transparency == Transparency::SemiTransparent {
result_mark = Some(mark);
} else {
break;
}
while let Some((mark, Transparency::SemiTransparent)) = iter.next() {
result_mark = Some(mark);
}
match result_mark {
Some(Some(call)) => Some(db.lookup_intern_macro_call(call.into()).def.krate),
Some(None) | None => None,
}
result_mark.flatten().map(|call| db.lookup_intern_macro_call(call.into()).def.krate)
}
pub use crate::name as __name;

109
crates/hir-expand/src/span.rs Normal file
View file

@ -0,0 +1,109 @@
use base_db::{
span::{ErasedFileAstId, SpanAnchor, SpanData, SyntaxContextId, ROOT_ERASED_FILE_AST_ID},
FileId,
};
use mbe::TokenMap;
use syntax::{ast::HasModuleItem, AstNode, TextRange, TextSize};
use triomphe::Arc;
use crate::db::ExpandDatabase;
pub type ExpansionSpanMap = TokenMap<SpanData>;
/// Spanmap for a macro file or a real file
#[derive(Clone, Debug)]
pub enum SpanMap {
/// Spanmap for a macro file
ExpansionSpanMap(Arc<ExpansionSpanMap>),
/// Spanmap for a real file
RealSpanMap(Arc<RealSpanMap>),
}
#[derive(Copy, Clone)]
pub enum SpanMapRef<'a> {
/// Spanmap for a macro file
ExpansionSpanMap(&'a ExpansionSpanMap),
/// Spanmap for a real file
RealSpanMap(&'a RealSpanMap),
}
impl mbe::SpanMapper<SpanData> for SpanMap {
fn span_for(&self, range: TextRange) -> SpanData {
self.span_for_range(range)
}
}
impl mbe::SpanMapper<SpanData> for SpanMapRef<'_> {
fn span_for(&self, range: TextRange) -> SpanData {
self.span_for_range(range)
}
}
impl mbe::SpanMapper<SpanData> for RealSpanMap {
fn span_for(&self, range: TextRange) -> SpanData {
self.span_for_range(range)
}
}
impl SpanMap {
pub fn span_for_range(&self, range: TextRange) -> SpanData {
match self {
Self::ExpansionSpanMap(span_map) => span_map.span_for_range(range),
Self::RealSpanMap(span_map) => span_map.span_for_range(range),
}
}
pub fn as_ref(&self) -> SpanMapRef<'_> {
match self {
Self::ExpansionSpanMap(span_map) => SpanMapRef::ExpansionSpanMap(span_map),
Self::RealSpanMap(span_map) => SpanMapRef::RealSpanMap(span_map),
}
}
}
impl SpanMapRef<'_> {
pub fn span_for_range(self, range: TextRange) -> SpanData {
match self {
Self::ExpansionSpanMap(span_map) => span_map.span_for_range(range),
Self::RealSpanMap(span_map) => span_map.span_for_range(range),
}
}
}
#[derive(PartialEq, Eq, Hash, Debug)]
pub struct RealSpanMap {
file_id: FileId,
/// Invariant: Sorted vec over TextSize
// FIXME: SortedVec<(TextSize, ErasedFileAstId)>?
pairs: Box<[(TextSize, ErasedFileAstId)]>,
}
impl RealSpanMap {
pub fn empty(file_id: FileId) -> Self {
RealSpanMap { file_id, pairs: Box::from([(TextSize::new(0), ROOT_ERASED_FILE_AST_ID)]) }
}
pub fn from_file(db: &dyn ExpandDatabase, file_id: FileId) -> Self {
let mut pairs = vec![(TextSize::new(0), ROOT_ERASED_FILE_AST_ID)];
let ast_id_map = db.ast_id_map(file_id.into());
pairs.extend(
db.parse(file_id)
.tree()
.items()
.map(|item| (item.syntax().text_range().start(), ast_id_map.ast_id(&item).erase())),
);
RealSpanMap { file_id, pairs: pairs.into_boxed_slice() }
}
pub fn span_for_range(&self, range: TextRange) -> SpanData {
let start = range.start();
let idx = self
.pairs
.binary_search_by(|&(it, _)| it.cmp(&start).then(std::cmp::Ordering::Less))
.unwrap_err();
let (offset, ast_id) = self.pairs[idx - 1];
SpanData {
range: range - offset,
anchor: SpanAnchor { file_id: self.file_id, ast_id },
ctx: SyntaxContextId::ROOT,
}
}
}