Use inventory for static ingredient registration (#934)

* use `inventory` for static ingredient registration

* remove unnecessary synchronization from memo tables

* use global ingredient caches for database-independent ingredients

* add manual ingredient registration API

* remove static ingredient index optimization when manual registration is in use

* fix atomic imports

* simplify ingredient caches

components/salsa-macro-rules/src/setup_accumulator_impl.rs

@@ -21,14 +21,21 @@ macro_rules! setup_accumulator_impl {
             use salsa::plumbing as $zalsa;
             use salsa::plumbing::accumulator as $zalsa_struct;
 
+            impl $zalsa::HasJar for $Struct {
+                type Jar = $zalsa_struct::JarImpl<$Struct>;
+                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
+            }
+
+            $zalsa::register_jar! {
+                $zalsa::ErasedJar::erase::<$Struct>()
+            }
+
             fn $ingredient(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<$Struct> {
                 static $CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Struct>> =
                     $zalsa::IngredientCache::new();
 
                 $CACHE.get_or_create(zalsa, || {
-                    zalsa
-                        .lookup_jar_by_type::<$zalsa_struct::JarImpl<$Struct>>()
-                        .get_or_create()
+                    zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Struct>>()
                 })
             }
 

components/salsa-macro-rules/src/setup_input_struct.rs

@@ -74,6 +74,15 @@ macro_rules! setup_input_struct {
 
             type $Configuration = $Struct;
 
+            impl $zalsa::HasJar for $Struct {
+                type Jar = $zalsa_struct::JarImpl<$Configuration>;
+                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
+            }
+
+            $zalsa::register_jar! {
+                $zalsa::ErasedJar::erase::<$Struct>()
+            }
+
             impl $zalsa_struct::Configuration for $Configuration {
                 const LOCATION: $zalsa::Location = $zalsa::Location {
                     file: file!(),
@@ -101,14 +110,14 @@ macro_rules! setup_input_struct {
                         $zalsa::IngredientCache::new();
 
                     CACHE.get_or_create(zalsa, || {
-                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
+                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
                     })
                 }
 
                 pub fn ingredient_mut(db: &mut dyn $zalsa::Database) -> (&mut $zalsa_struct::IngredientImpl<Self>, &mut $zalsa::Runtime) {
                     let zalsa_mut = db.zalsa_mut();
                     zalsa_mut.new_revision();
-                    let index = zalsa_mut.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create();
+                    let index = zalsa_mut.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>();
                     let (ingredient, runtime) = zalsa_mut.lookup_ingredient_mut(index);
                     let ingredient = ingredient.assert_type_mut::<$zalsa_struct::IngredientImpl<Self>>();
                     (ingredient, runtime)
@@ -149,8 +158,8 @@ macro_rules! setup_input_struct {
             impl $zalsa::SalsaStructInDb for $Struct {
                 type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
-                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
+                fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
                 }
 
                 #[inline]

components/salsa-macro-rules/src/setup_interned_struct.rs

@@ -92,6 +92,15 @@ macro_rules! setup_interned_struct {
 
             type $Configuration = $StructWithStatic;
 
+            impl<$($db_lt_arg)?> $zalsa::HasJar for $Struct<$($db_lt_arg)?> {
+                type Jar = $zalsa_struct::JarImpl<$Configuration>;
+                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
+            }
+
+            $zalsa::register_jar! {
+                $zalsa::ErasedJar::erase::<$StructWithStatic>()
+            }
+
             type $StructDataIdent<$db_lt> = ($($field_ty,)*);
 
             /// Key to use during hash lookups. Each field is some type that implements `Lookup<T>`
@@ -149,7 +158,7 @@ macro_rules! setup_interned_struct {
 
                     let zalsa = db.zalsa();
                     CACHE.get_or_create(zalsa, || {
-                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
+                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
                     })
                 }
             }
@@ -181,8 +190,8 @@ macro_rules! setup_interned_struct {
             impl< $($db_lt_arg)? > $zalsa::SalsaStructInDb for $Struct< $($db_lt_arg)? > {
                 type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
-                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
+                fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
                 }
 
                 #[inline]

components/salsa-macro-rules/src/setup_tracked_fn.rs

@@ -91,6 +91,16 @@ macro_rules! setup_tracked_fn {
 
             struct $Configuration;
 
+            $zalsa::register_jar! {
+                $zalsa::ErasedJar::erase::<$fn_name>()
+            }
+
+            #[allow(non_local_definitions)]
+            impl $zalsa::HasJar for $fn_name {
+                type Jar = $fn_name;
+                const KIND: $zalsa::JarKind = $zalsa::JarKind::TrackedFn;
+            }
+
             static $FN_CACHE: $zalsa::IngredientCache<$zalsa::function::IngredientImpl<$Configuration>> =
                 $zalsa::IngredientCache::new();
 
@@ -108,7 +118,7 @@ macro_rules! setup_tracked_fn {
                     impl $zalsa::SalsaStructInDb for $InternedData<'_> {
                         type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                        fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                        fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
                             $zalsa::IngredientIndices::empty()
                         }
 
@@ -155,27 +165,19 @@ macro_rules! setup_tracked_fn {
             impl $Configuration {
                 fn fn_ingredient(db: &dyn $Db) -> &$zalsa::function::IngredientImpl<$Configuration> {
                     let zalsa = db.zalsa();
-                    $FN_CACHE.get_or_create(zalsa, || {
-                        let jar_entry = zalsa.lookup_jar_by_type::<$Configuration>();
-
-                        // If the ingredient has already been inserted, we know that the downcaster
-                        // has also been registered. This is a fast-path for multi-database use cases
-                        // that bypass the ingredient cache and will always execute this closure.
-                        if let Some(index) = jar_entry.get() {
-                            return index;
-                        }
-
-                        <dyn $Db as $Db>::zalsa_register_downcaster(db);
-                        jar_entry.get_or_create()
-                    })
+                    $FN_CACHE
+                        .get_or_create(zalsa, || zalsa.lookup_jar_by_type::<$fn_name>())
+                        .get_or_init(|| <dyn $Db as $Db>::zalsa_register_downcaster(db))
                 }
 
                 pub fn fn_ingredient_mut(db: &mut dyn $Db) -> &mut $zalsa::function::IngredientImpl<Self> {
-                    <dyn $Db as $Db>::zalsa_register_downcaster(db);
+                    let view = <dyn $Db as $Db>::zalsa_register_downcaster(db);
                     let zalsa_mut = db.zalsa_mut();
-                    let index = zalsa_mut.lookup_jar_by_type::<$Configuration>().get_or_create();
+                    let index = zalsa_mut.lookup_jar_by_type::<$fn_name>();
                     let (ingredient, _) = zalsa_mut.lookup_ingredient_mut(index);
-                    ingredient.assert_type_mut::<$zalsa::function::IngredientImpl<Self>>()
+                    let ingredient = ingredient.assert_type_mut::<$zalsa::function::IngredientImpl<Self>>();
+                    ingredient.get_or_init(|| view);
+                    ingredient
                 }
 
                 $zalsa::macro_if! { $needs_interner =>
@@ -184,8 +186,7 @@ macro_rules! setup_tracked_fn {
                     ) -> &$zalsa::interned::IngredientImpl<$Configuration> {
                         let zalsa = db.zalsa();
                         $INTERN_CACHE.get_or_create(zalsa, || {
-                            <dyn $Db as $Db>::zalsa_register_downcaster(db);
-                            zalsa.lookup_jar_by_type::<$Configuration>().get_or_create().successor(0)
+                            zalsa.lookup_jar_by_type::<$fn_name>().successor(0)
                         })
                     }
                 }
@@ -248,42 +249,31 @@ macro_rules! setup_tracked_fn {
                 }
             }
 
-            impl $zalsa::Jar for $Configuration {
-                fn create_dependencies(zalsa: &$zalsa::Zalsa) -> $zalsa::IngredientIndices
-                where
-                    Self: Sized
-                {
-                    $zalsa::macro_if! {
-                        if $needs_interner {
-                            $zalsa::IngredientIndices::empty()
-                        } else {
-                            <$InternedData as $zalsa::SalsaStructInDb>::lookup_or_create_ingredient_index(zalsa)
-                        }
-                    }
-                }
-
+            #[allow(non_local_definitions)]
+            impl $zalsa::Jar for $fn_name {
                 fn create_ingredients(
-                    zalsa: &$zalsa::Zalsa,
+                    zalsa: &mut $zalsa::Zalsa,
                     first_index: $zalsa::IngredientIndex,
-                    struct_index: $zalsa::IngredientIndices,
                 ) -> Vec<Box<dyn $zalsa::Ingredient>> {
                     let struct_index: $zalsa::IngredientIndices = $zalsa::macro_if! {
                         if $needs_interner {
                             first_index.successor(0).into()
                         } else {
-                            struct_index
+                            // Note that struct ingredients are created before tracked functions,
+                            // so this cannot panic.
+                            <$InternedData as $zalsa::SalsaStructInDb>::lookup_ingredient_index(zalsa)
                         }
                     };
 
                     $zalsa::macro_if! { $needs_interner =>
-                        let intern_ingredient = <$zalsa::interned::IngredientImpl<$Configuration>>::new(
+                        let mut intern_ingredient = <$zalsa::interned::IngredientImpl<$Configuration>>::new(
                             first_index.successor(0)
                         );
                     }
 
                     let intern_ingredient_memo_types = $zalsa::macro_if! {
                         if $needs_interner {
-                            Some($zalsa::Ingredient::memo_table_types(&intern_ingredient))
+                            Some($zalsa::Ingredient::memo_table_types_mut(&mut intern_ingredient))
                         } else {
                             None
                         }
@@ -303,7 +293,6 @@ macro_rules! setup_tracked_fn {
                         first_index,
                         memo_ingredient_indices,
                         $lru,
-                        zalsa.views().downcaster_for::<dyn $Db>(),
                     );
                     $zalsa::macro_if! {
                         if $needs_interner {
@@ -386,6 +375,7 @@ macro_rules! setup_tracked_fn {
                 $zalsa::return_mode_expression!(($return_mode, __, __), $output_ty, result,)
             })
         }
+
         // The struct needs be last in the macro expansion in order to make the tracked
         // function's ident be identified as a function, not a struct, during semantic highlighting.
         // for more details, see https://github.com/salsa-rs/salsa/pull/612.

components/salsa-macro-rules/src/setup_tracked_struct.rs

@@ -107,8 +107,8 @@ macro_rules! setup_tracked_struct {
             std::marker::PhantomData<fn() -> &$db_lt ()>
         );
 
-        #[allow(clippy::all)]
         #[allow(dead_code)]
+        #[allow(clippy::all)]
         const _: () = {
             use salsa::plumbing as $zalsa;
             use $zalsa::tracked_struct as $zalsa_struct;
@@ -116,6 +116,15 @@ macro_rules! setup_tracked_struct {
 
             type $Configuration = $Struct<'static>;
 
+            impl<$db_lt> $zalsa::HasJar for $Struct<$db_lt> {
+                type Jar = $zalsa_struct::JarImpl<$Configuration>;
+                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
+            }
+
+            $zalsa::register_jar! {
+                $zalsa::ErasedJar::erase::<$Struct<'static>>()
+            }
+
             impl $zalsa_struct::Configuration for $Configuration {
                 const LOCATION: $zalsa::Location = $zalsa::Location {
                     file: file!(),
@@ -188,7 +197,7 @@ macro_rules! setup_tracked_struct {
                         $zalsa::IngredientCache::new();
 
                     CACHE.get_or_create(zalsa, || {
-                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
+                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
                     })
                 }
             }
@@ -210,8 +219,8 @@ macro_rules! setup_tracked_struct {
             impl $zalsa::SalsaStructInDb for $Struct<'_> {
                 type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
-                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
+                fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
                 }
 
                 #[inline]

components/salsa-macros/src/db.rs

@@ -110,7 +110,7 @@ impl DbMacro {
         let trait_name = &input.ident;
         input.items.push(parse_quote! {
             #[doc(hidden)]
-            fn zalsa_register_downcaster(&self);
+            fn zalsa_register_downcaster(&self) -> salsa::plumbing::DatabaseDownCaster<dyn #trait_name>;
         });
 
         let comment = format!(" Downcast a [`dyn Database`] to a [`dyn {trait_name}`]");
@@ -135,10 +135,11 @@ impl DbMacro {
         };
 
         input.items.push(parse_quote! {
+            #[cold]
+            #[inline(never)]
             #[doc(hidden)]
-            #[inline(always)]
-            fn zalsa_register_downcaster(&self)  {
-                salsa::plumbing::views(self).add(<Self as #TraitPath>::downcast);
+            fn zalsa_register_downcaster(&self) -> salsa::plumbing::DatabaseDownCaster<dyn #TraitPath> {
+                salsa::plumbing::views(self).add(<Self as #TraitPath>::downcast)
             }
         });
         input.items.push(parse_quote! {

components/salsa-macros/src/fn_util.rs

@@ -15,7 +15,7 @@ pub fn input_ids(hygiene: &Hygiene, sig: &syn::Signature, skip: usize) -> Vec<sy
                 }
             }
 
-            hygiene.ident(&format!("input{index}"))
+            hygiene.ident(format!("input{index}"))
         })
         .collect()
 }

components/salsa-macros/src/hygiene.rs

@@ -50,10 +50,10 @@ impl Hygiene {
     /// Generates an identifier similar to `text` but
     /// distinct from any identifiers that appear in the user's
     /// code.
-    pub(crate) fn ident(&self, text: &str) -> syn::Ident {
+    pub(crate) fn ident(&self, text: impl AsRef<str>) -> syn::Ident {
         // Make the default be `foo_` rather than `foo` -- this helps detect
         // cases where people wrote `foo` instead of `#foo` or `$foo` in the generated code.
-        let mut buffer = format!("{text}_");
+        let mut buffer = format!("{}_", text.as_ref());
 
         while self.user_tokens.contains(&buffer) {
             buffer.push('_');
@@ -61,4 +61,12 @@ impl Hygiene {
 
         syn::Ident::new(&buffer, proc_macro2::Span::call_site())
     }
+
+    /// Generates an identifier similar to `text` but distinct from any identifiers
+    /// that appear in the user's code.
+    ///
+    /// The identifier must be unique relative to the `scope` identifier.
+    pub(crate) fn scoped_ident(&self, scope: &syn::Ident, text: &str) -> syn::Ident {
+        self.ident(format!("{scope}_{text}"))
+    }
 }

components/salsa-macros/src/supertype.rs

@@ -72,8 +72,8 @@ fn enum_impl(enum_item: syn::ItemEnum) -> syn::Result<TokenStream> {
             type MemoIngredientMap = zalsa::MemoIngredientIndices;
 
             #[inline]
-            fn lookup_or_create_ingredient_index(__zalsa: &zalsa::Zalsa) -> zalsa::IngredientIndices {
-                zalsa::IngredientIndices::merge([ #( <#variant_types as zalsa::SalsaStructInDb>::lookup_or_create_ingredient_index(__zalsa) ),* ])
+            fn lookup_ingredient_index(__zalsa: &zalsa::Zalsa) -> zalsa::IngredientIndices {
+                zalsa::IngredientIndices::merge([ #( <#variant_types as zalsa::SalsaStructInDb>::lookup_ingredient_index(__zalsa) ),* ])
             }
 
             #[inline]

components/salsa-macros/src/tracked_fn.rs

@@ -132,10 +132,10 @@ impl Macro {
         inner_fn.sig.ident = self.hygiene.ident("inner");
 
         let zalsa = self.hygiene.ident("zalsa");
-        let Configuration = self.hygiene.ident("Configuration");
-        let InternedData = self.hygiene.ident("InternedData");
-        let FN_CACHE = self.hygiene.ident("FN_CACHE");
-        let INTERN_CACHE = self.hygiene.ident("INTERN_CACHE");
+        let Configuration = self.hygiene.scoped_ident(fn_name, "Configuration");
+        let InternedData = self.hygiene.scoped_ident(fn_name, "InternedData");
+        let FN_CACHE = self.hygiene.scoped_ident(fn_name, "FN_CACHE");
+        let INTERN_CACHE = self.hygiene.scoped_ident(fn_name, "INTERN_CACHE");
         let inner = &inner_fn.sig.ident;
 
         let function_type = function_type(&item);

components/salsa-macros/src/tracked_impl.rs

@@ -99,7 +99,7 @@ impl Macro {
         });
 
         let InnerTrait = self.hygiene.ident("InnerTrait");
-        let inner_fn_name = self.hygiene.ident(&fn_item.sig.ident.to_string());
+        let inner_fn_name = self.hygiene.ident(fn_item.sig.ident.to_string());
 
         let AssociatedFunctionArguments {
             self_token,