Merge #1784

1784: Support textual scoped macros r=matklad a=uHOOCCOOHu

Refactor the old simulation with `global_macro_scope`.

Now it is quite accurate to resolve textual scoped macros.
- Expand textual scoped macros in item and non-item place.
- Support `#[macro_use]` on `mod`.
- Textual scoped macros are collected into `nameres::ModuleScope`, so I think it makes #1727 easier to fix.
- It is implemented in a simple way to `clone()` current scoped macro ids into sub-modules. Though only indices are cloned, it will still increase some resolving time. Well, I've not bench-marked yet.

In my test with vscode extension, it can now successfully expand `dbg!` from `std` without `std::` prefix. "Goto definition" also works. Screenshot here:
<img width="281" alt="Screenshot_20190907_043442" src="https://user-images.githubusercontent.com/14816024/64458794-ddb47900-d128-11e9-95e3-1c8569978825.png">


Co-authored-by: uHOOCCOOHu <hooccooh1896@gmail.com>

crates/ra_hir/src/nameres.rs

@@ -138,7 +138,21 @@ pub(crate) struct ModuleData {
 #[derive(Debug, Default, PartialEq, Eq, Clone)]
 pub struct ModuleScope {
     items: FxHashMap<Name, Resolution>,
+    /// Macros in current module scoped
+    ///
+    /// This scope works exactly the same way that item scoping does.
+    /// Macro invocation with quantified path will search in it.
+    /// See details below.
     macros: FxHashMap<Name, MacroDef>,
+    /// Macros visable in current module in legacy textual scope
+    ///
+    /// For macros invoked by an unquatified identifier like `bar!()`, `legacy_macros` will be searched in first.
+    /// If it yields no result, then it turns to module scoped `macros`.
+    /// It macros with name quatified with a path like `crate::foo::bar!()`, `legacy_macros` will be skipped,
+    /// and only normal scoped `macros` will be searched in.
+    ///
+    /// Note that this automatically inherit macros defined textually before the definition of module itself.
+    legacy_macros: FxHashMap<Name, MacroDef>,
 }
 
 static BUILTIN_SCOPE: Lazy<FxHashMap<Name, Resolution>> = Lazy::new(|| {
@@ -164,6 +178,7 @@ impl ModuleScope {
             _ => None,
         })
     }
+    /// It resolves in module scope. Textual scoped macros are ignored here.
     fn get_item_or_macro(&self, name: &Name) -> Option<ItemOrMacro> {
         match (self.get(name), self.macros.get(name)) {
             (Some(item), _) if !item.def.is_none() => Some(Either::A(item.def)),
@@ -171,6 +186,9 @@ impl ModuleScope {
             _ => None,
         }
     }
+    fn get_legacy_macro(&self, name: &Name) -> Option<MacroDef> {
+        self.legacy_macros.get(name).copied()
+    }
 }
 
 type ItemOrMacro = Either<PerNs<ModuleDef>, MacroDef>;
@@ -484,16 +502,21 @@ impl CrateDefMap {
         name: &Name,
     ) -> ItemOrMacro {
         // Resolve in:
+        //  - legacy scope
         //  - current module / scope
         //  - extern prelude
         //  - std prelude
+        let from_legacy_macro = self[module]
+            .scope
+            .get_legacy_macro(name)
+            .map_or_else(|| Either::A(PerNs::none()), Either::B);
         let from_scope =
             self[module].scope.get_item_or_macro(name).unwrap_or_else(|| Either::A(PerNs::none()));
         let from_extern_prelude =
             self.extern_prelude.get(name).map_or(PerNs::none(), |&it| PerNs::types(it));
         let from_prelude = self.resolve_in_prelude(db, name);
 
-        or(from_scope, or(Either::A(from_extern_prelude), from_prelude))
+        or(from_legacy_macro, or(from_scope, or(Either::A(from_extern_prelude), from_prelude)))
     }
 
     fn resolve_name_in_extern_prelude(&self, name: &Name) -> PerNs<ModuleDef> {

crates/ra_hir/src/nameres/collector.rs

@@ -14,8 +14,8 @@ use crate::{
         raw, CrateDefMap, CrateModuleId, ItemOrMacro, ModuleData, ModuleDef, PerNs,
         ReachedFixedPoint, Resolution, ResolveMode,
     },
-    AstId, Const, Enum, Function, HirFileId, MacroDef, Module, Name, Path, Static, Struct, Trait,
+    AstId, Const, Enum, Function, HirFileId, MacroDef, Module, Name, Path, PathKind, Static,
-    TypeAlias, Union,
+    Struct, Trait, TypeAlias, Union,
 };
 
 pub(super) fn collect_defs(db: &impl DefDatabase, mut def_map: CrateDefMap) -> CrateDefMap {
@@ -40,7 +40,6 @@ pub(super) fn collect_defs(db: &impl DefDatabase, mut def_map: CrateDefMap) -> C
         glob_imports: FxHashMap::default(),
         unresolved_imports: Vec::new(),
         unexpanded_macros: Vec::new(),
-        global_macro_scope: FxHashMap::default(),
         macro_stack_monitor: MacroStackMonitor::default(),
     };
     collector.collect();
@@ -82,7 +81,6 @@ struct DefCollector<DB> {
     glob_imports: FxHashMap<CrateModuleId, Vec<(CrateModuleId, raw::ImportId)>>,
     unresolved_imports: Vec<(CrateModuleId, raw::ImportId, raw::ImportData)>,
     unexpanded_macros: Vec<(CrateModuleId, AstId<ast::MacroCall>, Path)>,
-    global_macro_scope: FxHashMap<Name, MacroDefId>,
 
     /// Some macro use `$tt:tt which mean we have to handle the macro perfectly
     /// To prevent stack overflow, we add a deep counter here for prevent that.
@@ -136,20 +134,6 @@ where
         macro_id: MacroDefId,
         export: bool,
     ) {
-        // macro-by-example in Rust have completely weird name resolution logic,
-        // unlike anything else in the language. We'd don't fully implement yet,
-        // just give a somewhat precise approximation.
-        //
-        // Specifically, we store a set of visible macros in each module, just
-        // like how we do with usual items. This is wrong, however, because
-        // macros can be shadowed and their scopes are mostly unrelated to
-        // modules. To paper over the second problem, we also maintain
-        // `global_macro_scope` which works when we construct `CrateDefMap`, but
-        // is completely ignored in expressions.
-        //
-        // What we should do is that, in CrateDefMap, we should maintain a
-        // separate tower of macro scopes, with ids. Then, for each item in the
-        // module, we need to store it's macro scope.
         let def = Either::B(MacroDef { id: macro_id });
 
         // In Rust, `#[macro_export]` macros are unconditionally visible at the
@@ -162,13 +146,29 @@ where
             self.def_map.exported_macros.insert(name.clone(), macro_id);
         }
         self.update(module_id, None, &[(name.clone(), def)]);
-        self.global_macro_scope.insert(name, macro_id);
+        self.define_legacy_macro(module_id, name.clone(), macro_id);
+    }
+
+    /// Define a legacy textual scoped macro in module
+    ///
+    /// We use a map `legacy_macros` to store all legacy textual scoped macros visable per module.
+    /// It will clone all macros from parent legacy scope, whose definition is prior to
+    /// the definition of current module.
+    /// And also, `macro_use` on a module will import all legacy macros visable inside to
+    /// current legacy scope, with possible shadowing.
+    fn define_legacy_macro(&mut self, module_id: CrateModuleId, name: Name, macro_id: MacroDefId) {
+        // Always shadowing
+        self.def_map.modules[module_id].scope.legacy_macros.insert(name, MacroDef { id: macro_id });
     }
 
     /// Import macros from `#[macro_use] extern crate`.
     ///
     /// They are non-scoped, and will only be inserted into mutable `global_macro_scope`.
-    fn import_macros_from_extern_crate(&mut self, import: &raw::ImportData) {
+    fn import_macros_from_extern_crate(
+        &mut self,
+        current_module_id: CrateModuleId,
+        import: &raw::ImportData,
+    ) {
         log::debug!(
             "importing macros from extern crate: {:?} ({:?})",
             import,
@@ -184,14 +184,14 @@ where
 
         if let Some(ModuleDef::Module(m)) = res.take_types() {
             tested_by!(macro_rules_from_other_crates_are_visible_with_macro_use);
-            self.import_all_macros_exported(m);
+            self.import_all_macros_exported(current_module_id, m);
         }
     }
 
-    fn import_all_macros_exported(&mut self, module: Module) {
+    fn import_all_macros_exported(&mut self, current_module_id: CrateModuleId, module: Module) {
         let item_map = self.db.crate_def_map(module.krate);
         for (name, &macro_id) in &item_map.exported_macros {
-            self.global_macro_scope.insert(name.clone(), macro_id);
+            self.define_legacy_macro(current_module_id, name.clone(), macro_id);
         }
     }
 
@@ -528,7 +528,7 @@ where
         if let Some(prelude_module) = self.def_collector.def_map.prelude {
             if prelude_module.krate != self.def_collector.def_map.krate {
                 tested_by!(prelude_is_macro_use);
-                self.def_collector.import_all_macros_exported(prelude_module);
+                self.def_collector.import_all_macros_exported(self.module_id, prelude_module);
             }
         }
 
@@ -539,7 +539,7 @@ where
             if let raw::RawItem::Import(import_id) = *item {
                 let import = self.raw_items[import_id].clone();
                 if import.is_extern_crate && import.is_macro_use {
-                    self.def_collector.import_macros_from_extern_crate(&import);
+                    self.def_collector.import_macros_from_extern_crate(self.module_id, &import);
                 }
             }
         }
@@ -561,10 +561,11 @@ where
     fn collect_module(&mut self, module: &raw::ModuleData) {
         match module {
             // inline module, just recurse
-            raw::ModuleData::Definition { name, items, ast_id, attr_path } => {
+            raw::ModuleData::Definition { name, items, ast_id, attr_path, is_macro_use } => {
                 let module_id =
                     self.push_child_module(name.clone(), ast_id.with_file_id(self.file_id), None);
                 let parent_module = ParentModule { name, attr_path: attr_path.as_ref() };
+
                 ModCollector {
                     def_collector: &mut *self.def_collector,
                     module_id,
@@ -573,9 +574,12 @@ where
                     parent_module: Some(parent_module),
                 }
                 .collect(&*items);
+                if *is_macro_use {
+                    self.import_all_legacy_macros(module_id);
+                }
             }
             // out of line module, resolve, parse and recurse
-            raw::ModuleData::Declaration { name, ast_id, attr_path } => {
+            raw::ModuleData::Declaration { name, ast_id, attr_path, is_macro_use } => {
                 let ast_id = ast_id.with_file_id(self.file_id);
                 let is_root = self.def_collector.def_map.modules[self.module_id].parent.is_none();
                 match resolve_submodule(
@@ -596,7 +600,10 @@ where
                             raw_items: &raw_items,
                             parent_module: None,
                         }
-                        .collect(raw_items.items())
+                        .collect(raw_items.items());
+                        if *is_macro_use {
+                            self.import_all_legacy_macros(module_id);
+                        }
                     }
                     Err(candidate) => self.def_collector.def_map.diagnostics.push(
                         DefDiagnostic::UnresolvedModule {
@@ -621,6 +628,7 @@ where
         modules[res].parent = Some(self.module_id);
         modules[res].declaration = Some(declaration);
         modules[res].definition = definition;
+        modules[res].scope.legacy_macros = modules[self.module_id].scope.legacy_macros.clone();
         modules[self.module_id].children.insert(name.clone(), res);
         let resolution = Resolution {
             def: PerNs::types(
@@ -674,20 +682,32 @@ where
 
         let ast_id = mac.ast_id.with_file_id(self.file_id);
 
-        // Case 2: try to expand macro_rules from this crate, triggering
+        // Case 2: try to resolve in legacy scope and expand macro_rules, triggering
         // recursive item collection.
-        if let Some(macro_id) =
+        if let Some(macro_def) = mac.path.as_ident().and_then(|name| {
-            mac.path.as_ident().and_then(|name| self.def_collector.global_macro_scope.get(&name))
+            self.def_collector.def_map[self.module_id].scope.get_legacy_macro(&name)
-        {
+        }) {
-            let def = *macro_id;
+            let def = macro_def.id;
             let macro_call_id = MacroCallLoc { def, ast_id }.id(self.def_collector.db);
 
             self.def_collector.collect_macro_expansion(self.module_id, macro_call_id, def);
             return;
         }
 
-        // Case 3: path to a macro from another crate, expand during name resolution
+        // Case 3: resolve in module scope, expand during name resolution.
-        self.def_collector.unexpanded_macros.push((self.module_id, ast_id, mac.path.clone()))
+        // We rewrite simple path `macro_name` to `self::macro_name` to force resolve in module scope only.
+        let mut path = mac.path.clone();
+        if path.is_ident() {
+            path.kind = PathKind::Self_;
+        }
+        self.def_collector.unexpanded_macros.push((self.module_id, ast_id, path));
+    }
+
+    fn import_all_legacy_macros(&mut self, module_id: CrateModuleId) {
+        let macros = self.def_collector.def_map[module_id].scope.legacy_macros.clone();
+        for (name, macro_) in macros {
+            self.def_collector.define_legacy_macro(self.module_id, name.clone(), macro_.id);
+        }
     }
 }
 
@@ -715,7 +735,6 @@ mod tests {
             glob_imports: FxHashMap::default(),
             unresolved_imports: Vec::new(),
             unexpanded_macros: Vec::new(),
-            global_macro_scope: FxHashMap::default(),
             macro_stack_monitor: monitor,
         };
         collector.collect();

crates/ra_hir/src/nameres/raw.rs

@@ -134,12 +134,14 @@ pub(super) enum ModuleData {
         name: Name,
         ast_id: FileAstId<ast::Module>,
         attr_path: Option<SmolStr>,
+        is_macro_use: bool,
     },
     Definition {
         name: Name,
         ast_id: FileAstId<ast::Module>,
         items: Vec<RawItem>,
         attr_path: Option<SmolStr>,
+        is_macro_use: bool,
     },
 }
 
@@ -267,10 +269,15 @@ impl RawItemsCollector {
         };
 
         let ast_id = self.source_ast_id_map.ast_id(&module);
+        let is_macro_use = module.has_atom_attr("macro_use");
         if module.has_semi() {
             let attr_path = extract_mod_path_attribute(&module);
-            let item =
+            let item = self.raw_items.modules.alloc(ModuleData::Declaration {
-                self.raw_items.modules.alloc(ModuleData::Declaration { name, ast_id, attr_path });
+                name,
+                ast_id,
+                attr_path,
+                is_macro_use,
+            });
             self.push_item(current_module, RawItem::Module(item));
             return;
         }
@@ -282,6 +289,7 @@ impl RawItemsCollector {
                 ast_id,
                 items: Vec::new(),
                 attr_path,
+                is_macro_use,
             });
             self.process_module(Some(item), item_list);
             self.push_item(current_module, RawItem::Module(item));

crates/ra_hir/src/nameres/tests/macros.rs

@@ -268,12 +268,114 @@ fn prelude_cycle() {
     );
     assert_snapshot!(map, @r###"
         ⋮crate
-        ⋮foo: t
         ⋮prelude: t
         ⋮
         ⋮crate::prelude
         ⋮declare_mod: m
-        ⋮
+    "###);
-        ⋮crate::foo
+}
+
+#[test]
+fn plain_macros_are_legacy_textual_scoped() {
+    let map = def_map(
+        r#"
+        //- /main.rs
+        mod m1;
+        bar!(NotFoundNotMacroUse);
+
+        mod m2 {
+            foo!(NotFoundBeforeInside2);
+        }
+
+        macro_rules! foo {
+            ($x:ident) => { struct $x; }
+        }
+        foo!(Ok);
+
+        mod m3;
+        foo!(OkShadowStop);
+        bar!(NotFoundMacroUseStop);
+
+        #[macro_use]
+        mod m5 {
+            #[macro_use]
+            mod m6 {
+                macro_rules! foo {
+                    ($x:ident) => { fn $x() {} }
+                }
+            }
+        }
+        foo!(ok_double_macro_use_shadow);
+
+        baz!(NotFoundBefore);
+        #[macro_use]
+        mod m7 {
+            macro_rules! baz {
+                ($x:ident) => { struct $x; }
+            }
+        }
+        baz!(OkAfter);
+
+        //- /m1.rs
+        foo!(NotFoundBeforeInside1);
+        macro_rules! bar {
+            ($x:ident) => { struct $x; }
+        }
+
+        //- /m3/mod.rs
+        foo!(OkAfterInside);
+        macro_rules! foo {
+            ($x:ident) => { fn $x() {} }
+        }
+        foo!(ok_shadow);
+
+        #[macro_use]
+        mod m4;
+        bar!(OkMacroUse);
+
+        //- /m3/m4.rs
+        foo!(ok_shadow_deep);
+        macro_rules! bar {
+            ($x:ident) => { struct $x; }
+        }
+        "#,
+    );
+    assert_snapshot!(map, @r###"
+   ⋮crate
+   ⋮Ok: t v
+   ⋮OkAfter: t v
+   ⋮OkShadowStop: t v
+   ⋮foo: m
+   ⋮m1: t
+   ⋮m2: t
+   ⋮m3: t
+   ⋮m5: t
+   ⋮m7: t
+   ⋮ok_double_macro_use_shadow: v
+   ⋮
+   ⋮crate::m7
+   ⋮baz: m
+   ⋮
+   ⋮crate::m1
+   ⋮bar: m
+   ⋮
+   ⋮crate::m5
+   ⋮m6: t
+   ⋮
+   ⋮crate::m5::m6
+   ⋮foo: m
+   ⋮
+   ⋮crate::m2
+   ⋮
+   ⋮crate::m3
+   ⋮OkAfterInside: t v
+   ⋮OkMacroUse: t v
+   ⋮foo: m
+   ⋮m4: t
+   ⋮ok_shadow: v
+   ⋮
+   ⋮crate::m3::m4
+   ⋮bar: m
+   ⋮ok_shadow_deep: v
     "###);
 }

crates/ra_hir/src/ty/tests.rs

@@ -2803,6 +2803,41 @@ fn main() {
     );
 }
 
+#[test]
+fn infer_legacy_textual_scoped_macros_expanded() {
+    assert_snapshot!(
+        infer(r#"
+struct Foo(Vec<i32>);
+
+#[macro_use]
+mod m {
+    macro_rules! foo {
+        ($($item:expr),*) => {
+            {
+                Foo(vec![$($item,)*])
+            }
+        };
+    }
+}
+
+fn main() {
+    let x = foo!(1,2);
+    let y = crate::foo!(1,2);
+}
+"#),
+        @r###"
+    ![0; 17) '{Foo(v...,2,])}': Foo
+    ![1; 4) 'Foo': Foo({unknown}) -> Foo
+    ![1; 16) 'Foo(vec![1,2,])': Foo
+    ![5; 15) 'vec![1,2,]': {unknown}
+    [195; 251) '{     ...,2); }': ()
+    [205; 206) 'x': Foo
+    [228; 229) 'y': {unknown}
+    [232; 248) 'crate:...!(1,2)': {unknown}
+    "###
+    );
+}
+
 #[ignore]
 #[test]
 fn method_resolution_trait_before_autoref() {