Auto merge of #15959 - Veykril:macro-shower3, r=lnicola

TokenMap -> SpanMap rewrite

Opening early so I can have an overview over the full diff more easily, still very unfinished and lots of work to be done.

The gist of what this PR does is move away from assigning IDs to tokens in arguments and expansions and instead gives the subtrees the text ranges they are sourced from (made relative to some item for incrementality). This means we now only have a single map per expension, opposed to map for expansion and arguments.

A few of the things that are not done yet (in arbitrary order):
- [x] generally clean up the current mess
- [x] proc-macros, have been completely ignored so far
- [x] syntax fixups, has been commented out for the time being needs to be rewritten on top of some marker SyntaxContextId
- [x] macro invocation syntax contexts are not properly passed around yet, so $crate hygiene does not work in all cases (but most)
  - [x] builtin macros do not set spans properly, $crate basically does not work with them rn (which we use)
~~- [ ] remove all uses of dummy spans (or if that does not work, change the dummy entries for dummy spans so that tests will not silently pass due to havin a file id for the dummy file)~~
  - [x] de-queryfy `macro_expand`, the sole caller of it is `parse_macro_expansion`, and both of these are lru-cached with the same limit so having it be a query is pointless
- [x] docs and more docs
- [x] fix eager macro spans and other stuff
  - [x] simplify include! handling
- [x] Figure out how to undo the sudden `()` expression wrapping in expansions / alternatively prioritize getting invisible delimiters working again
- [x] Simplify InFile stuff and HirFIleId extensions
~~- [ ] span crate containing all the file ids, span stuff, ast ids. Then remove the dependency injection generics from tt and mbe~~

Fixes https://github.com/rust-lang/rust-analyzer/issues/10300
Fixes https://github.com/rust-lang/rust-analyzer/issues/15685
This commit is contained in:
bors 2023-12-04 19:59:53 +00:00
commit e91fdf7860
133 changed files with 5403 additions and 4205 deletions

View file

@ -1,5 +1,6 @@
//! Attributes & documentation for hir types.
use base_db::FileId;
use hir_def::{
attr::AttrsWithOwner,
item_scope::ItemInNs,
@ -8,7 +9,10 @@ use hir_def::{
resolver::{HasResolver, Resolver, TypeNs},
AssocItemId, AttrDefId, ModuleDefId,
};
use hir_expand::{hygiene::Hygiene, name::Name};
use hir_expand::{
name::Name,
span::{RealSpanMap, SpanMapRef},
};
use hir_ty::db::HirDatabase;
use syntax::{ast, AstNode};
@ -234,7 +238,11 @@ fn modpath_from_str(db: &dyn HirDatabase, link: &str) -> Option<ModPath> {
if ast_path.syntax().text() != link {
return None;
}
ModPath::from_src(db.upcast(), ast_path, &Hygiene::new_unhygienic())
ModPath::from_src(
db.upcast(),
ast_path,
SpanMapRef::RealSpanMap(&RealSpanMap::absolute(FileId(0))),
)
};
let full = try_get_modpath(link);

View file

@ -3,10 +3,27 @@
//! we didn't do that.
//!
//! But we need this for at least LRU caching at the query level.
pub use hir_def::db::*;
pub use hir_def::db::{
AttrsQuery, BlockDefMapQuery, BlockItemTreeQueryQuery, BodyQuery, BodyWithSourceMapQuery,
ConstDataQuery, ConstVisibilityQuery, CrateDefMapQueryQuery, CrateLangItemsQuery,
CrateSupportsNoStdQuery, DefDatabase, DefDatabaseStorage, EnumDataQuery,
EnumDataWithDiagnosticsQuery, ExprScopesQuery, ExternCrateDeclDataQuery,
FieldVisibilitiesQuery, FieldsAttrsQuery, FieldsAttrsSourceMapQuery, FileItemTreeQuery,
FunctionDataQuery, FunctionVisibilityQuery, GenericParamsQuery, ImplDataQuery,
ImplDataWithDiagnosticsQuery, ImportMapQuery, InternAnonymousConstQuery, InternBlockQuery,
InternConstQuery, InternDatabase, InternDatabaseStorage, InternEnumQuery,
InternExternBlockQuery, InternExternCrateQuery, InternFunctionQuery, InternImplQuery,
InternInTypeConstQuery, InternMacro2Query, InternMacroRulesQuery, InternProcMacroQuery,
InternStaticQuery, InternStructQuery, InternTraitAliasQuery, InternTraitQuery,
InternTypeAliasQuery, InternUnionQuery, InternUseQuery, LangAttrQuery, LangItemQuery,
Macro2DataQuery, MacroRulesDataQuery, ProcMacroDataQuery, StaticDataQuery, StructDataQuery,
StructDataWithDiagnosticsQuery, TraitAliasDataQuery, TraitDataQuery,
TraitDataWithDiagnosticsQuery, TypeAliasDataQuery, UnionDataQuery,
UnionDataWithDiagnosticsQuery, VariantsAttrsQuery, VariantsAttrsSourceMapQuery,
};
pub use hir_expand::db::{
AstIdMapQuery, DeclMacroExpanderQuery, ExpandDatabase, ExpandDatabaseStorage,
ExpandProcMacroQuery, HygieneFrameQuery, InternMacroCallQuery, MacroArgNodeQuery,
MacroExpandQuery, ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery,
ExpandProcMacroQuery, InternMacroCallQuery, InternSyntaxContextQuery, MacroArgQuery,
ParseMacroExpansionErrorQuery, ParseMacroExpansionQuery, RealSpanMapQuery,
};
pub use hir_ty::db::*;

View file

@ -124,7 +124,7 @@ pub use {
hir_expand::{
attrs::{Attr, AttrId},
name::{known, Name},
ExpandResult, HirFileId, InFile, MacroFile, Origin,
tt, ExpandResult, HirFileId, HirFileIdExt, InFile, InMacroFile, InRealFile, MacroFileId,
},
hir_ty::{
display::{ClosureStyle, HirDisplay, HirDisplayError, HirWrite},
@ -140,7 +140,10 @@ pub use {
#[allow(unused)]
use {
hir_def::path::Path,
hir_expand::{hygiene::Hygiene, name::AsName},
hir_expand::{
name::AsName,
span::{ExpansionSpanMap, RealSpanMap, SpanMap, SpanMapRef},
},
};
/// hir::Crate describes a single crate. It's the main interface with which
@ -3490,9 +3493,34 @@ impl Impl {
self.id.lookup(db.upcast()).container.into()
}
pub fn as_builtin_derive(self, db: &dyn HirDatabase) -> Option<InFile<ast::Attr>> {
pub fn as_builtin_derive_path(self, db: &dyn HirDatabase) -> Option<InMacroFile<ast::Path>> {
let src = self.source(db)?;
src.file_id.as_builtin_derive_attr_node(db.upcast())
let macro_file = src.file_id.macro_file()?;
let loc = db.lookup_intern_macro_call(macro_file.macro_call_id);
let (derive_attr, derive_index) = match loc.kind {
MacroCallKind::Derive { ast_id, derive_attr_index, derive_index } => {
let module_id = self.id.lookup(db.upcast()).container;
(
db.crate_def_map(module_id.krate())[module_id.local_id]
.scope
.derive_macro_invoc(ast_id, derive_attr_index)?,
derive_index,
)
}
_ => return None,
};
let file_id = MacroFileId { macro_call_id: derive_attr };
let path = db
.parse_macro_expansion(file_id)
.value
.0
.syntax_node()
.children()
.nth(derive_index as usize)
.and_then(<ast::Attr as AstNode>::cast)
.and_then(|it| it.path())?;
Some(InMacroFile { file_id, value: path })
}
pub fn check_orphan_rules(self, db: &dyn HirDatabase) -> bool {

View file

@ -15,10 +15,14 @@ use hir_def::{
type_ref::Mutability,
AsMacroCall, DefWithBodyId, FieldId, FunctionId, MacroId, TraitId, VariantId,
};
use hir_expand::{db::ExpandDatabase, name::AsName, ExpansionInfo, MacroCallId};
use hir_expand::{
db::ExpandDatabase, files::InRealFile, name::AsName, ExpansionInfo, HirFileIdExt, MacroCallId,
MacroFileId, MacroFileIdExt,
};
use itertools::Itertools;
use rustc_hash::{FxHashMap, FxHashSet};
use smallvec::{smallvec, SmallVec};
use stdx::TupleExt;
use syntax::{
algo::skip_trivia_token,
ast::{self, HasAttrs as _, HasGenericParams, HasLoopBody},
@ -114,11 +118,11 @@ pub struct Semantics<'db, DB> {
pub struct SemanticsImpl<'db> {
pub db: &'db dyn HirDatabase,
s2d_cache: RefCell<SourceToDefCache>,
expansion_info_cache: RefCell<FxHashMap<HirFileId, Option<ExpansionInfo>>>,
// Rootnode to HirFileId cache
expansion_info_cache: RefCell<FxHashMap<MacroFileId, ExpansionInfo>>,
/// Rootnode to HirFileId cache
cache: RefCell<FxHashMap<SyntaxNode, HirFileId>>,
// MacroCall to its expansion's HirFileId cache
macro_call_cache: RefCell<FxHashMap<InFile<ast::MacroCall>, HirFileId>>,
/// MacroCall to its expansion's MacroFileId cache
macro_call_cache: RefCell<FxHashMap<InFile<ast::MacroCall>, MacroFileId>>,
}
impl<DB> fmt::Debug for Semantics<'_, DB> {
@ -255,7 +259,7 @@ impl<'db> SemanticsImpl<'db> {
pub fn expand(&self, macro_call: &ast::MacroCall) -> Option<SyntaxNode> {
let sa = self.analyze_no_infer(macro_call.syntax())?;
let file_id = sa.expand(self.db, InFile::new(sa.file_id, macro_call))?;
let node = self.parse_or_expand(file_id);
let node = self.parse_or_expand(file_id.into());
Some(node)
}
@ -524,52 +528,54 @@ impl<'db> SemanticsImpl<'db> {
res
}
// FIXME: should only take real file inputs for simplicity
fn descend_into_macros_impl(
&self,
token: SyntaxToken,
// FIXME: We might want this to be Option<TextSize> to be able to opt out of subrange
// mapping, specifically for node downmapping
offset: TextSize,
_offset: TextSize,
f: &mut dyn FnMut(InFile<SyntaxToken>) -> bool,
) {
// FIXME: Clean this up
let _p = profile::span("descend_into_macros");
let relative_token_offset = token.text_range().start().checked_sub(offset);
let parent = match token.parent() {
let sa = match token.parent().and_then(|parent| self.analyze_no_infer(&parent)) {
Some(it) => it,
None => return,
};
let sa = match self.analyze_no_infer(&parent) {
Some(it) => it,
None => return,
};
let def_map = sa.resolver.def_map();
let mut stack: SmallVec<[_; 4]> = smallvec![InFile::new(sa.file_id, token)];
let mut cache = self.expansion_info_cache.borrow_mut();
let mut mcache = self.macro_call_cache.borrow_mut();
let span = match sa.file_id.repr() {
base_db::span::HirFileIdRepr::FileId(file_id) => {
self.db.real_span_map(file_id).span_for_range(token.text_range())
}
base_db::span::HirFileIdRepr::MacroFile(macro_file) => cache
.entry(macro_file)
.or_insert_with(|| macro_file.expansion_info(self.db.upcast()))
.exp_map
.span_at(token.text_range().start()),
};
let def_map = sa.resolver.def_map();
let mut stack: SmallVec<[_; 4]> = smallvec![InFile::new(sa.file_id, token)];
let mut process_expansion_for_token =
|stack: &mut SmallVec<_>, macro_file, item, token: InFile<&_>| {
|stack: &mut SmallVec<_>, macro_file, _token: InFile<&_>| {
let expansion_info = cache
.entry(macro_file)
.or_insert_with(|| macro_file.expansion_info(self.db.upcast()))
.as_ref()?;
.or_insert_with(|| macro_file.expansion_info(self.db.upcast()));
{
let InFile { file_id, value } = expansion_info.expanded();
self.cache(value, file_id);
}
let mapped_tokens = expansion_info.map_token_down(
self.db.upcast(),
item,
token,
relative_token_offset,
)?;
let mapped_tokens = expansion_info.map_range_down(span, None)?;
let len = stack.len();
// requeue the tokens we got from mapping our current token down
stack.extend(mapped_tokens);
stack.extend(mapped_tokens.map(Into::into));
// if the length changed we have found a mapping for the token
(stack.len() != len).then_some(())
};
@ -578,26 +584,21 @@ impl<'db> SemanticsImpl<'db> {
// either due to not being in a macro-call or because its unused push it into the result vec,
// otherwise push the remapped tokens back into the queue as they can potentially be remapped again.
while let Some(token) = stack.pop() {
self.db.unwind_if_cancelled();
let was_not_remapped = (|| {
// First expand into attribute invocations
let containing_attribute_macro_call = self.with_ctx(|ctx| {
token.value.parent_ancestors().filter_map(ast::Item::cast).find_map(|item| {
if item.attrs().next().is_none() {
// Don't force populate the dyn cache for items that don't have an attribute anyways
return None;
}
Some((ctx.item_to_macro_call(token.with_value(item.clone()))?, item))
Some(ctx.item_to_macro_call(token.with_value(item.clone()))?)
})
});
if let Some((call_id, item)) = containing_attribute_macro_call {
let file_id = call_id.as_file();
return process_expansion_for_token(
&mut stack,
file_id,
Some(item),
token.as_ref(),
);
if let Some(call_id) = containing_attribute_macro_call {
let file_id = call_id.as_macro_file();
return process_expansion_for_token(&mut stack, file_id, token.as_ref());
}
// Then check for token trees, that means we are either in a function-like macro or
@ -613,7 +614,8 @@ impl<'db> SemanticsImpl<'db> {
}
if let Some(macro_call) = ast::MacroCall::cast(parent.clone()) {
let mcall = token.with_value(macro_call);
let mcall: hir_expand::files::InFileWrapper<HirFileId, ast::MacroCall> =
token.with_value(macro_call);
let file_id = match mcache.get(&mcall) {
Some(&it) => it,
None => {
@ -622,7 +624,7 @@ impl<'db> SemanticsImpl<'db> {
it
}
};
process_expansion_for_token(&mut stack, file_id, None, token.as_ref())
process_expansion_for_token(&mut stack, file_id, token.as_ref())
} else if let Some(meta) = ast::Meta::cast(parent) {
// attribute we failed expansion for earlier, this might be a derive invocation
// or derive helper attribute
@ -643,11 +645,10 @@ impl<'db> SemanticsImpl<'db> {
match derive_call {
Some(call_id) => {
// resolved to a derive
let file_id = call_id.as_file();
let file_id = call_id.as_macro_file();
return process_expansion_for_token(
&mut stack,
file_id,
Some(adt.into()),
token.as_ref(),
);
}
@ -679,13 +680,11 @@ impl<'db> SemanticsImpl<'db> {
let id = self.db.ast_id_map(token.file_id).ast_id(&adt);
let helpers =
def_map.derive_helpers_in_scope(InFile::new(token.file_id, id))?;
let item = Some(adt.into());
let mut res = None;
for (.., derive) in helpers.iter().filter(|(helper, ..)| *helper == attr_name) {
res = res.or(process_expansion_for_token(
&mut stack,
derive.as_file(),
item.clone(),
derive.as_macro_file(),
token.as_ref(),
));
}
@ -737,14 +736,16 @@ impl<'db> SemanticsImpl<'db> {
pub fn original_range_opt(&self, node: &SyntaxNode) -> Option<FileRange> {
let node = self.find_file(node);
node.original_file_range_opt(self.db.upcast())
.filter(|(_, ctx)| ctx.is_root())
.map(TupleExt::head)
}
/// Attempts to map the node out of macro expanded files.
/// This only work for attribute expansions, as other ones do not have nodes as input.
pub fn original_ast_node<N: AstNode>(&self, node: N) -> Option<N> {
self.wrap_node_infile(node).original_ast_node(self.db.upcast()).map(
|InFile { file_id, value }| {
self.cache(find_root(value.syntax()), file_id);
|InRealFile { file_id, value }| {
self.cache(find_root(value.syntax()), file_id.into());
value
},
)
@ -755,8 +756,8 @@ impl<'db> SemanticsImpl<'db> {
pub fn original_syntax_node(&self, node: &SyntaxNode) -> Option<SyntaxNode> {
let InFile { file_id, .. } = self.find_file(node);
InFile::new(file_id, node).original_syntax_node(self.db.upcast()).map(
|InFile { file_id, value }| {
self.cache(find_root(&value), file_id);
|InRealFile { file_id, value }| {
self.cache(find_root(&value), file_id.into());
value
},
)
@ -851,9 +852,9 @@ impl<'db> SemanticsImpl<'db> {
pub fn resolve_trait(&self, path: &ast::Path) -> Option<Trait> {
let analyze = self.analyze(path.syntax())?;
let hygiene = hir_expand::hygiene::Hygiene::new(self.db.upcast(), analyze.file_id);
let ctx = LowerCtx::with_hygiene(self.db.upcast(), &hygiene);
let hir_path = Path::from_src(path.clone(), &ctx)?;
let span_map = self.db.span_map(analyze.file_id);
let ctx = LowerCtx::with_span_map(self.db.upcast(), span_map);
let hir_path = Path::from_src(&ctx, path.clone())?;
match analyze.resolver.resolve_path_in_type_ns_fully(self.db.upcast(), &hir_path)? {
TypeNs::TraitId(id) => Some(Trait { id }),
_ => None,
@ -1037,7 +1038,7 @@ impl<'db> SemanticsImpl<'db> {
fn with_ctx<F: FnOnce(&mut SourceToDefCtx<'_, '_>) -> T, T>(&self, f: F) -> T {
let mut cache = self.s2d_cache.borrow_mut();
let mut ctx = SourceToDefCtx { db: self.db, cache: &mut cache };
let mut ctx = SourceToDefCtx { db: self.db, dynmap_cache: &mut cache };
f(&mut ctx)
}
@ -1451,7 +1452,7 @@ impl SemanticsScope<'_> {
/// necessary a heuristic, as it doesn't take hygiene into account.
pub fn speculative_resolve(&self, path: &ast::Path) -> Option<PathResolution> {
let ctx = LowerCtx::with_file_id(self.db.upcast(), self.file_id);
let path = Path::from_src(path.clone(), &ctx)?;
let path = Path::from_src(&ctx, path.clone())?;
resolve_hir_path(self.db, &self.resolver, &path)
}

View file

@ -97,7 +97,7 @@ use hir_def::{
FunctionId, GenericDefId, GenericParamId, ImplId, LifetimeParamId, MacroId, ModuleId, StaticId,
StructId, TraitAliasId, TraitId, TypeAliasId, TypeParamId, UnionId, UseId, VariantId,
};
use hir_expand::{attrs::AttrId, name::AsName, HirFileId, MacroCallId};
use hir_expand::{attrs::AttrId, name::AsName, HirFileId, HirFileIdExt, MacroCallId};
use rustc_hash::FxHashMap;
use smallvec::SmallVec;
use stdx::{impl_from, never};
@ -112,7 +112,7 @@ pub(super) type SourceToDefCache = FxHashMap<(ChildContainer, HirFileId), DynMap
pub(super) struct SourceToDefCtx<'a, 'b> {
pub(super) db: &'b dyn HirDatabase,
pub(super) cache: &'a mut SourceToDefCache,
pub(super) dynmap_cache: &'a mut SourceToDefCache,
}
impl SourceToDefCtx<'_, '_> {
@ -300,7 +300,7 @@ impl SourceToDefCtx<'_, '_> {
fn cache_for(&mut self, container: ChildContainer, file_id: HirFileId) -> &DynMap {
let db = self.db;
self.cache
self.dynmap_cache
.entry((container, file_id))
.or_insert_with(|| container.child_by_source(db, file_id))
}

View file

@ -26,11 +26,10 @@ use hir_def::{
};
use hir_expand::{
builtin_fn_macro::BuiltinFnLikeExpander,
hygiene::Hygiene,
mod_path::path,
name,
name::{AsName, Name},
HirFileId, InFile,
HirFileId, HirFileIdExt, InFile, MacroFileId, MacroFileIdExt,
};
use hir_ty::{
diagnostics::{
@ -484,7 +483,7 @@ impl SourceAnalyzer {
macro_call: InFile<&ast::MacroCall>,
) -> Option<Macro> {
let ctx = LowerCtx::with_file_id(db.upcast(), macro_call.file_id);
let path = macro_call.value.path().and_then(|ast| Path::from_src(ast, &ctx))?;
let path = macro_call.value.path().and_then(|ast| Path::from_src(&ctx, ast))?;
self.resolver
.resolve_path_as_macro(db.upcast(), path.mod_path()?, Some(MacroSubNs::Bang))
.map(|(it, _)| it.into())
@ -596,9 +595,8 @@ impl SourceAnalyzer {
}
// This must be a normal source file rather than macro file.
let hygiene = Hygiene::new(db.upcast(), self.file_id);
let ctx = LowerCtx::with_hygiene(db.upcast(), &hygiene);
let hir_path = Path::from_src(path.clone(), &ctx)?;
let ctx = LowerCtx::with_span_map(db.upcast(), db.span_map(self.file_id));
let hir_path = Path::from_src(&ctx, path.clone())?;
// Case where path is a qualifier of a use tree, e.g. foo::bar::{Baz, Qux} where we are
// trying to resolve foo::bar.
@ -755,14 +753,15 @@ impl SourceAnalyzer {
&self,
db: &dyn HirDatabase,
macro_call: InFile<&ast::MacroCall>,
) -> Option<HirFileId> {
) -> Option<MacroFileId> {
let krate = self.resolver.krate();
let macro_call_id = macro_call.as_call_id(db.upcast(), krate, |path| {
self.resolver
.resolve_path_as_macro(db.upcast(), &path, Some(MacroSubNs::Bang))
.map(|(it, _)| macro_id_to_def_id(db.upcast(), it))
})?;
Some(macro_call_id.as_file()).filter(|it| it.expansion_level(db.upcast()) < 64)
// why the 64?
Some(macro_call_id.as_macro_file()).filter(|it| it.expansion_level(db.upcast()) < 64)
}
pub(crate) fn resolve_variant(

View file

@ -50,13 +50,8 @@ impl DeclarationLocation {
node.as_ref().original_file_range(db.upcast())
}
pub fn original_name_range(&self, db: &dyn HirDatabase) -> Option<FileRange> {
if let Some(file_id) = self.hir_file_id.file_id() {
// fast path to prevent parsing
return Some(FileRange { file_id, range: self.name_ptr.text_range() });
}
let node = resolve_node(db, self.hir_file_id, &self.name_ptr);
node.as_ref().original_file_range_opt(db.upcast())
pub fn original_name_range(&self, db: &dyn HirDatabase) -> FileRange {
InFile::new(self.hir_file_id, self.name_ptr.text_range()).original_file_range(db.upcast())
}
}