language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-10-02 22:54:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Auto merge of #15959 - Veykril:macro-shower3, r=lnicola

Browse source 
TokenMap -> SpanMap rewrite

Opening early so I can have an overview over the full diff more easily, still very unfinished and lots of work to be done.

The gist of what this PR does is move away from assigning IDs to tokens in arguments and expansions and instead gives the subtrees the text ranges they are sourced from (made relative to some item for incrementality). This means we now only have a single map per expension, opposed to map for expansion and arguments.

A few of the things that are not done yet (in arbitrary order):
- [x] generally clean up the current mess
- [x] proc-macros, have been completely ignored so far
- [x] syntax fixups, has been commented out for the time being needs to be rewritten on top of some marker SyntaxContextId
- [x] macro invocation syntax contexts are not properly passed around yet, so $crate hygiene does not work in all cases (but most)
  - [x] builtin macros do not set spans properly, $crate basically does not work with them rn (which we use)
~~- [ ] remove all uses of dummy spans (or if that does not work, change the dummy entries for dummy spans so that tests will not silently pass due to havin a file id for the dummy file)~~
  - [x] de-queryfy `macro_expand`, the sole caller of it is `parse_macro_expansion`, and both of these are lru-cached with the same limit so having it be a query is pointless
- [x] docs and more docs
- [x] fix eager macro spans and other stuff
  - [x] simplify include! handling
- [x] Figure out how to undo the sudden `()` expression wrapping in expansions / alternatively prioritize getting invisible delimiters working again
- [x] Simplify InFile stuff and HirFIleId extensions
~~- [ ] span crate containing all the file ids, span stuff, ast ids. Then remove the dependency injection generics from tt and mbe~~

Fixes https://github.com/rust-lang/rust-analyzer/issues/10300
Fixes https://github.com/rust-lang/rust-analyzer/issues/15685
This commit is contained in:
bors 2023-12-04 19:59:53 +00:00
commit e91fdf7860
133 changed files with 5403 additions and 4205 deletions
Download patch file Download diff file Expand all files Collapse all files

143
crates/hir-def/src/item_tree/lower.rs
View file

@ -2,12 +2,13 @@
use std::collections::hash_map::Entry;
use hir_expand::{ast_id_map::AstIdMap, hygiene::Hygiene, HirFileId};
use hir_expand::{ast_id_map::AstIdMap, span::SpanMapRef, HirFileId};
use syntax::ast::{self, HasModuleItem, HasTypeBounds};
use crate::{
generics::{GenericParams, TypeParamData, TypeParamProvenance},
type_ref::{LifetimeRef, TraitBoundModifier, TraitRef},
LocalLifetimeParamId, LocalTypeOrConstParamId,
};
use super::*;
@ -33,8 +34,8 @@ impl<'a> Ctx<'a> {
}
}
pub(super) fn hygiene(&self) -> &Hygiene {
self.body_ctx.hygiene()
pub(super) fn span_map(&self) -> SpanMapRef<'_> {
self.body_ctx.span_map()
}
pub(super) fn lower_module_items(mut self, item_owner: &dyn HasModuleItem) -> ItemTree {
@ -79,7 +80,7 @@ impl<'a> Ctx<'a> {
pub(super) fn lower_block(mut self, block: &ast::BlockExpr) -> ItemTree {
self.tree
.attrs
.insert(AttrOwner::TopLevel, RawAttrs::new(self.db.upcast(), block, self.hygiene()));
.insert(AttrOwner::TopLevel, RawAttrs::new(self.db.upcast(), block, self.span_map()));
self.tree.top_level = block
.statements()
.filter_map(|stmt| match stmt {
@ -109,8 +110,7 @@ impl<'a> Ctx<'a> {
}
fn lower_mod_item(&mut self, item: &ast::Item) -> Option<ModItem> {
let attrs = RawAttrs::new(self.db.upcast(), item, self.hygiene());
let item: ModItem = match item {
let mod_item: ModItem = match item {
ast::Item::Struct(ast) => self.lower_struct(ast)?.into(),
ast::Item::Union(ast) => self.lower_union(ast)?.into(),
ast::Item::Enum(ast) => self.lower_enum(ast)?.into(),
@ -129,10 +129,10 @@ impl<'a> Ctx<'a> {
ast::Item::MacroDef(ast) => self.lower_macro_def(ast)?.into(),
ast::Item::ExternBlock(ast) => self.lower_extern_block(ast).into(),
};
let attrs = RawAttrs::new(self.db.upcast(), item, self.span_map());
self.add_attrs(mod_item.into(), attrs);
self.add_attrs(item.into(), attrs);
Some(item)
Some(mod_item)
}
fn add_attrs(&mut self, item: AttrOwner, attrs: RawAttrs) {
@ -146,21 +146,32 @@ impl<'a> Ctx<'a> {
}
}
fn lower_assoc_item(&mut self, item: &ast::AssocItem) -> Option<AssocItem> {
match item {
fn lower_assoc_item(&mut self, item_node: &ast::AssocItem) -> Option<AssocItem> {
let item: AssocItem = match item_node {
ast::AssocItem::Fn(ast) => self.lower_function(ast).map(Into::into),
ast::AssocItem::TypeAlias(ast) => self.lower_type_alias(ast).map(Into::into),
ast::AssocItem::Const(ast) => Some(self.lower_const(ast).into()),
ast::AssocItem::MacroCall(ast) => self.lower_macro_call(ast).map(Into::into),
}
}?;
let attrs = RawAttrs::new(self.db.upcast(), item_node, self.span_map());
self.add_attrs(
match item {
AssocItem::Function(it) => AttrOwner::ModItem(ModItem::Function(it)),
AssocItem::TypeAlias(it) => AttrOwner::ModItem(ModItem::TypeAlias(it)),
AssocItem::Const(it) => AttrOwner::ModItem(ModItem::Const(it)),
AssocItem::MacroCall(it) => AttrOwner::ModItem(ModItem::MacroCall(it)),
},
attrs,
);
Some(item)
}
fn lower_struct(&mut self, strukt: &ast::Struct) -> Option<FileItemTreeId<Struct>> {
let visibility = self.lower_visibility(strukt);
let name = strukt.name()?.as_name();
let ast_id = self.source_ast_id_map.ast_id(strukt);
let generic_params = self.lower_generic_params(HasImplicitSelf::No, strukt);
let fields = self.lower_fields(&strukt.kind());
let ast_id = self.source_ast_id_map.ast_id(strukt);
let res = Struct { name, visibility, generic_params, fields, ast_id };
Some(id(self.data().structs.alloc(res)))
}
@ -184,7 +195,10 @@ impl<'a> Ctx<'a> {
for field in fields.fields() {
if let Some(data) = self.lower_record_field(&field) {
let idx = self.data().fields.alloc(data);
self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &field, self.hygiene()));
self.add_attrs(
idx.into(),
RawAttrs::new(self.db.upcast(), &field, self.span_map()),
);
}
}
let end = self.next_field_idx();
@ -205,7 +219,7 @@ impl<'a> Ctx<'a> {
for (i, field) in fields.fields().enumerate() {
let data = self.lower_tuple_field(i, &field);
let idx = self.data().fields.alloc(data);
self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &field, self.hygiene()));
self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &field, self.span_map()));
}
let end = self.next_field_idx();
IdxRange::new(start..end)
@ -222,12 +236,12 @@ impl<'a> Ctx<'a> {
fn lower_union(&mut self, union: &ast::Union) -> Option<FileItemTreeId<Union>> {
let visibility = self.lower_visibility(union);
let name = union.name()?.as_name();
let ast_id = self.source_ast_id_map.ast_id(union);
let generic_params = self.lower_generic_params(HasImplicitSelf::No, union);
let fields = match union.record_field_list() {
Some(record_field_list) => self.lower_fields(&StructKind::Record(record_field_list)),
None => Fields::Record(IdxRange::new(self.next_field_idx()..self.next_field_idx())),
};
let ast_id = self.source_ast_id_map.ast_id(union);
let res = Union { name, visibility, generic_params, fields, ast_id };
Some(id(self.data().unions.alloc(res)))
}
@ -235,12 +249,12 @@ impl<'a> Ctx<'a> {
fn lower_enum(&mut self, enum_: &ast::Enum) -> Option<FileItemTreeId<Enum>> {
let visibility = self.lower_visibility(enum_);
let name = enum_.name()?.as_name();
let ast_id = self.source_ast_id_map.ast_id(enum_);
let generic_params = self.lower_generic_params(HasImplicitSelf::No, enum_);
let variants = match &enum_.variant_list() {
Some(variant_list) => self.lower_variants(variant_list),
None => IdxRange::new(self.next_variant_idx()..self.next_variant_idx()),
};
let ast_id = self.source_ast_id_map.ast_id(enum_);
let res = Enum { name, visibility, generic_params, variants, ast_id };
Some(id(self.data().enums.alloc(res)))
}
@ -252,7 +266,7 @@ impl<'a> Ctx<'a> {
let idx = self.data().variants.alloc(data);
self.add_attrs(
idx.into(),
RawAttrs::new(self.db.upcast(), &variant, self.hygiene()),
RawAttrs::new(self.db.upcast(), &variant, self.span_map()),
);
}
}
@ -303,28 +317,29 @@ impl<'a> Ctx<'a> {
});
self.add_attrs(
idx.into(),
RawAttrs::new(self.db.upcast(), &self_param, self.hygiene()),
RawAttrs::new(self.db.upcast(), &self_param, self.span_map()),
);
has_self_param = true;
}
for param in param_list.params() {
let ast_id = self.source_ast_id_map.ast_id(&param);
let idx = match param.dotdotdot_token() {
Some(_) => {
let ast_id = self.source_ast_id_map.ast_id(&param);
self.data()
.params
.alloc(Param { type_ref: None, ast_id: ParamAstId::Param(ast_id) })
}
Some(_) => self
.data()
.params
.alloc(Param { type_ref: None, ast_id: ParamAstId::Param(ast_id) }),
None => {
let type_ref = TypeRef::from_ast_opt(&self.body_ctx, param.ty());
let ty = Interned::new(type_ref);
let ast_id = self.source_ast_id_map.ast_id(&param);
self.data()
.params
.alloc(Param { type_ref: Some(ty), ast_id: ParamAstId::Param(ast_id) })
}
};
self.add_attrs(idx.into(), RawAttrs::new(self.db.upcast(), &param, self.hygiene()));
self.add_attrs(
idx.into(),
RawAttrs::new(self.db.upcast(), &param, self.span_map()),
);
}
}
let end_param = self.next_param_idx();
@ -394,8 +409,8 @@ impl<'a> Ctx<'a> {
let type_ref = type_alias.ty().map(|it| self.lower_type_ref(&it));
let visibility = self.lower_visibility(type_alias);
let bounds = self.lower_type_bounds(type_alias);
let generic_params = self.lower_generic_params(HasImplicitSelf::No, type_alias);
let ast_id = self.source_ast_id_map.ast_id(type_alias);
let generic_params = self.lower_generic_params(HasImplicitSelf::No, type_alias);
let res = TypeAlias { name, visibility, bounds, generic_params, type_ref, ast_id };
Some(id(self.data().type_aliases.alloc(res)))
}
@ -443,23 +458,17 @@ impl<'a> Ctx<'a> {
fn lower_trait(&mut self, trait_def: &ast::Trait) -> Option<FileItemTreeId<Trait>> {
let name = trait_def.name()?.as_name();
let visibility = self.lower_visibility(trait_def);
let ast_id = self.source_ast_id_map.ast_id(trait_def);
let generic_params =
self.lower_generic_params(HasImplicitSelf::Yes(trait_def.type_bound_list()), trait_def);
let is_auto = trait_def.auto_token().is_some();
let is_unsafe = trait_def.unsafe_token().is_some();
let ast_id = self.source_ast_id_map.ast_id(trait_def);
let items = trait_def
.assoc_item_list()
.into_iter()
.flat_map(|list| list.assoc_items())
.filter_map(|item| {
let attrs = RawAttrs::new(self.db.upcast(), &item, self.hygiene());
self.lower_assoc_item(&item).map(|item| {
self.add_attrs(ModItem::from(item).into(), attrs);
item
})
})
.filter_map(|item_node| self.lower_assoc_item(&item_node))
.collect();
let def = Trait { name, visibility, generic_params, is_auto, is_unsafe, items, ast_id };
@ -472,17 +481,18 @@ impl<'a> Ctx<'a> {
) -> Option<FileItemTreeId<TraitAlias>> {
let name = trait_alias_def.name()?.as_name();
let visibility = self.lower_visibility(trait_alias_def);
let ast_id = self.source_ast_id_map.ast_id(trait_alias_def);
let generic_params = self.lower_generic_params(
HasImplicitSelf::Yes(trait_alias_def.type_bound_list()),
trait_alias_def,
);
let ast_id = self.source_ast_id_map.ast_id(trait_alias_def);
let alias = TraitAlias { name, visibility, generic_params, ast_id };
Some(id(self.data().trait_aliases.alloc(alias)))
}
fn lower_impl(&mut self, impl_def: &ast::Impl) -> Option<FileItemTreeId<Impl>> {
let ast_id = self.source_ast_id_map.ast_id(impl_def);
// Note that trait impls don't get implicit `Self` unlike traits, because here they are a
// type alias rather than a type parameter, so this is handled by the resolver.
let generic_params = self.lower_generic_params(HasImplicitSelf::No, impl_def);
@ -499,14 +509,8 @@ impl<'a> Ctx<'a> {
.assoc_item_list()
.into_iter()
.flat_map(|it| it.assoc_items())
.filter_map(|item| {
let assoc = self.lower_assoc_item(&item)?;
let attrs = RawAttrs::new(self.db.upcast(), &item, self.hygiene());
self.add_attrs(ModItem::from(assoc).into(), attrs);
Some(assoc)
})
.filter_map(|item| self.lower_assoc_item(&item))
.collect();
let ast_id = self.source_ast_id_map.ast_id(impl_def);
let res =
Impl { generic_params, target_trait, self_ty, is_negative, is_unsafe, items, ast_id };
Some(id(self.data().impls.alloc(res)))
@ -515,7 +519,7 @@ impl<'a> Ctx<'a> {
fn lower_use(&mut self, use_item: &ast::Use) -> Option<FileItemTreeId<Use>> {
let visibility = self.lower_visibility(use_item);
let ast_id = self.source_ast_id_map.ast_id(use_item);
let (use_tree, _) = lower_use_tree(self.db, self.hygiene(), use_item.use_tree()?)?;
let (use_tree, _) = lower_use_tree(self.db, self.span_map(), use_item.use_tree()?)?;
let res = Use { visibility, ast_id, use_tree };
Some(id(self.data().uses.alloc(res)))
@ -537,10 +541,16 @@ impl<'a> Ctx<'a> {
}
fn lower_macro_call(&mut self, m: &ast::MacroCall) -> Option<FileItemTreeId<MacroCall>> {
let path = Interned::new(ModPath::from_src(self.db.upcast(), m.path()?, self.hygiene())?);
let span_map = self.span_map();
let path = Interned::new(ModPath::from_src(self.db.upcast(), m.path()?, span_map)?);
let ast_id = self.source_ast_id_map.ast_id(m);
let expand_to = hir_expand::ExpandTo::from_call_site(m);
let res = MacroCall { path, ast_id, expand_to };
let res = MacroCall {
path,
ast_id,
expand_to,
call_site: span_map.span_for_range(m.syntax().text_range()).ctx,
};
Some(id(self.data().macro_calls.alloc(res)))
}
@ -572,15 +582,15 @@ impl<'a> Ctx<'a> {
// (in other words, the knowledge that they're in an extern block must not be used).
// This is because an extern block can contain macros whose ItemTree's top-level items
// should be considered to be in an extern block too.
let attrs = RawAttrs::new(self.db.upcast(), &item, self.hygiene());
let id: ModItem = match item {
ast::ExternItem::Fn(ast) => self.lower_function(&ast)?.into(),
ast::ExternItem::Static(ast) => self.lower_static(&ast)?.into(),
ast::ExternItem::TypeAlias(ty) => self.lower_type_alias(&ty)?.into(),
ast::ExternItem::MacroCall(call) => self.lower_macro_call(&call)?.into(),
let mod_item: ModItem = match &item {
ast::ExternItem::Fn(ast) => self.lower_function(ast)?.into(),
ast::ExternItem::Static(ast) => self.lower_static(ast)?.into(),
ast::ExternItem::TypeAlias(ty) => self.lower_type_alias(ty)?.into(),
ast::ExternItem::MacroCall(call) => self.lower_macro_call(call)?.into(),
};
self.add_attrs(id.into(), attrs);
Some(id)
let attrs = RawAttrs::new(self.db.upcast(), &item, self.span_map());
self.add_attrs(mod_item.into(), attrs);
Some(mod_item)
})
.collect()
});
@ -612,12 +622,16 @@ impl<'a> Ctx<'a> {
generics.fill_bounds(&self.body_ctx, bounds, Either::Left(self_param));
}
let add_param_attrs = |item, param| {
let attrs = RawAttrs::new(self.db.upcast(), &param, self.body_ctx.hygiene());
let add_param_attrs = |item: Either<LocalTypeOrConstParamId, LocalLifetimeParamId>,
param| {
let attrs = RawAttrs::new(self.db.upcast(), &param, self.body_ctx.span_map());
// This is identical to the body of `Ctx::add_attrs()` but we can't call that here
// because it requires `&mut self` and the call to `generics.fill()` below also
// references `self`.
match self.tree.attrs.entry(item) {
match self.tree.attrs.entry(match item {
Either::Right(id) => id.into(),
Either::Left(id) => id.into(),
}) {
Entry::Occupied(mut entry) => {
*entry.get_mut() = entry.get().merge(attrs);
}
@ -643,7 +657,8 @@ impl<'a> Ctx<'a> {
}
fn lower_visibility(&mut self, item: &dyn ast::HasVisibility) -> RawVisibilityId {
let vis = RawVisibility::from_ast_with_hygiene(self.db, item.visibility(), self.hygiene());
let vis =
RawVisibility::from_ast_with_span_map(self.db, item.visibility(), self.span_map());
self.data().vis.alloc(vis)
}
@ -721,7 +736,7 @@ fn lower_abi(abi: ast::Abi) -> Interned<str> {
struct UseTreeLowering<'a> {
db: &'a dyn DefDatabase,
hygiene: &'a Hygiene,
span_map: SpanMapRef<'a>,
mapping: Arena<ast::UseTree>,
}
@ -734,7 +749,7 @@ impl UseTreeLowering<'_> {
// E.g. `use something::{inner}` (prefix is `None`, path is `something`)
// or `use something::{path::{inner::{innerer}}}` (prefix is `something::path`, path is `inner`)
Some(path) => {
match ModPath::from_src(self.db.upcast(), path, self.hygiene) {
match ModPath::from_src(self.db.upcast(), path, self.span_map) {
Some(it) => Some(it),
None => return None, // FIXME: report errors somewhere
}
@ -753,7 +768,7 @@ impl UseTreeLowering<'_> {
} else {
let is_glob = tree.star_token().is_some();
let path = match tree.path() {
Some(path) => Some(ModPath::from_src(self.db.upcast(), path, self.hygiene)?),
Some(path) => Some(ModPath::from_src(self.db.upcast(), path, self.span_map)?),
None => None,
};
let alias = tree.rename().map(|a| {
@ -789,10 +804,10 @@ impl UseTreeLowering<'_> {
pub(crate) fn lower_use_tree(
db: &dyn DefDatabase,
hygiene: &Hygiene,
span_map: SpanMapRef<'_>,
tree: ast::UseTree,
) -> Option<(UseTree, Arena<ast::UseTree>)> {
let mut lowering = UseTreeLowering { db, hygiene, mapping: Arena::new() };
let mut lowering = UseTreeLowering { db, span_map, mapping: Arena::new() };
let tree = lowering.lower_use_tree(tree)?;
Some((tree, lowering.mapping))
}