.github/workflows/test.yml

@@ -55,10 +55,10 @@ jobs:
         run: cargo clippy --workspace --all-targets -- -D warnings
       - name: Test
         run: cargo nextest run --workspace --all-targets --no-fail-fast
-      - name: Test Manual Registration / no-default-features
-        run: cargo nextest run --workspace --tests --no-fail-fast --no-default-features --features macros
       - name: Test docs
         run: cargo test --workspace --doc
+      - name: Check (without default features)
+        run: cargo check --workspace --no-default-features
 
   miri:
     name: Miri

Cargo.toml

@@ -13,35 +13,31 @@ salsa-macro-rules = { version = "0.23.0", path = "components/salsa-macro-rules"
 salsa-macros = { version = "0.23.0", path = "components/salsa-macros", optional = true }
 
 boxcar = "0.2.13"
-crossbeam-queue = "0.3.12"
+crossbeam-queue = "0.3.11"
 crossbeam-utils = "0.8.21"
 hashbrown = "0.15"
 hashlink = "0.10"
 indexmap = "2"
 intrusive-collections = "0.9.7"
+papaya = "0.2.2"
 parking_lot = "0.12"
 portable-atomic = "1"
 rustc-hash = "2"
 smallvec = "1"
 tracing = { version = "0.1", default-features = false, features = ["std"] }
 
-# Automatic ingredient registration.
-inventory = { version = "0.3.20", optional = true }
-
 # parallel map
 rayon = { version = "1.10.0", optional = true }
 
 # Stuff we want Update impls for by default
 compact_str = { version = "0.9", optional = true }
-thin-vec = "0.2.14"
+thin-vec = "0.2.13"
 
-shuttle = { version = "0.8.1", optional = true }
+shuttle = { version = "0.8.0", optional = true }
 
 [features]
-default = ["salsa_unstable", "rayon", "macros", "inventory", "accumulator"]
+default = ["salsa_unstable", "rayon", "macros"]
-inventory = ["dep:inventory"]
 shuttle = ["dep:shuttle"]
-accumulator = ["salsa-macro-rules/accumulator"]
 # FIXME: remove `salsa_unstable` before 1.0.
 salsa_unstable = []
 macros = ["dep:salsa-macros"]
@@ -55,18 +51,18 @@ salsa-macros = { version = "=0.23.0", path = "components/salsa-macros" }
 
 [dev-dependencies]
 # examples
-crossbeam-channel = "0.5.15"
+crossbeam-channel = "0.5.14"
 dashmap = { version = "6", features = ["raw-api"] }
-eyre = "0.6.12"
+eyre = "0.6.8"
 notify-debouncer-mini = "0.4.1"
-ordered-float = "5.0.0"
+ordered-float = "4.2.1"
 
 # tests/benches
 annotate-snippets = "0.11.5"
-codspeed-criterion-compat = { version = "3.0.5", default-features = false }
+codspeed-criterion-compat = { version = "2.6.0", default-features = false }
-expect-test = "1.5.1"
+expect-test = "1.5.0"
 rustversion = "1.0"
-test-log = { version = "0.2.18", features = ["trace"] }
+test-log = { version = "0.2.11", features = ["trace"] }
 trybuild = "1.0"
 
 [target.'cfg(all(not(target_os = "windows"), not(target_os = "openbsd"), any(target_arch = "x86_64", target_arch = "aarch64", target_arch = "powerpc64")))'.dev-dependencies]
@@ -83,20 +79,11 @@ harness = false
 [[bench]]
 name = "accumulator"
 harness = false
-required-features = ["accumulator"]
 
 [[bench]]
 name = "dataflow"
 harness = false
 
-[[example]]
-name = "lazy-input"
-required-features = ["accumulator"]
-
-[[example]]
-name = "calc"
-required-features = ["accumulator"]
-
 [workspace]
 members = ["components/salsa-macro-rules", "components/salsa-macros"]
 

components/salsa-macro-rules/Cargo.toml

@@ -9,6 +9,3 @@ rust-version.workspace = true
 description = "Declarative macros for the salsa crate"
 
 [dependencies]
-
-[features]
-accumulator = []

components/salsa-macro-rules/src/gate_accumulated.rs

@@ -1,13 +0,0 @@
-#[cfg(feature = "accumulator")]
-#[macro_export]
-macro_rules! gate_accumulated {
-    ($($body:tt)*) => {
-        $($body)*
-    };
-}
-
-#[cfg(not(feature = "accumulator"))]
-#[macro_export]
-macro_rules! gate_accumulated {
-    ($($body:tt)*) => {};
-}

components/salsa-macro-rules/src/lib.rs

@@ -12,12 +12,10 @@
 //! from a submodule is to use multiple crates, hence the existence
 //! of this crate.
 
-mod gate_accumulated;
 mod macro_if;
 mod maybe_backdate;
 mod maybe_default;
 mod return_mode;
-#[cfg(feature = "accumulator")]
 mod setup_accumulator_impl;
 mod setup_input_struct;
 mod setup_interned_struct;

components/salsa-macro-rules/src/setup_accumulator_impl.rs

@@ -21,26 +21,15 @@ macro_rules! setup_accumulator_impl {
             use salsa::plumbing as $zalsa;
             use salsa::plumbing::accumulator as $zalsa_struct;
 
-            impl $zalsa::HasJar for $Struct {
-                type Jar = $zalsa_struct::JarImpl<$Struct>;
-                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
-            }
-
-            $zalsa::register_jar! {
-                $zalsa::ErasedJar::erase::<$Struct>()
-            }
-
             fn $ingredient(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<$Struct> {
                 static $CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Struct>> =
                     $zalsa::IngredientCache::new();
 
-                // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the only
+                $CACHE.get_or_create(zalsa, || {
-                // ingredient created by our jar is the struct ingredient.
+                    zalsa
-                unsafe {
+                        .lookup_jar_by_type::<$zalsa_struct::JarImpl<$Struct>>()
-                    $CACHE.get_or_create(zalsa, || {
+                        .get_or_create()
-                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Struct>>()
+                })
-                    })
-                }
             }
 
             impl $zalsa::Accumulator for $Struct {

components/salsa-macro-rules/src/setup_input_struct.rs

@@ -50,9 +50,6 @@ macro_rules! setup_input_struct {
         // If true, generate a debug impl.
         generate_debug_impl: $generate_debug_impl:tt,
 
-        // The function used to implement `C::heap_size`.
-        heap_size_fn: $($heap_size_fn:path)?,
-
         // Annoyingly macro-rules hygiene does not extend to items defined in the macro.
         // We have the procedural macro generate names for those items that are
         // not used elsewhere in the user's code.
@@ -77,15 +74,6 @@ macro_rules! setup_input_struct {
 
             type $Configuration = $Struct;
 
-            impl $zalsa::HasJar for $Struct {
-                type Jar = $zalsa_struct::JarImpl<$Configuration>;
-                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
-            }
-
-            $zalsa::register_jar! {
-                $zalsa::ErasedJar::erase::<$Struct>()
-            }
-
             impl $zalsa_struct::Configuration for $Configuration {
                 const LOCATION: $zalsa::Location = $zalsa::Location {
                     file: file!(),
@@ -101,12 +89,6 @@ macro_rules! setup_input_struct {
 
                 type Revisions = [$zalsa::Revision; $N];
                 type Durabilities = [$zalsa::Durability; $N];
-
-                $(
-                    fn heap_size(value: &Self::Fields) -> Option<usize> {
-                        Some($heap_size_fn(value))
-                    }
-                )?
             }
 
             impl $Configuration {
@@ -118,18 +100,15 @@ macro_rules! setup_input_struct {
                     static CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Configuration>> =
                         $zalsa::IngredientCache::new();
 
-                    // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the only
+                    CACHE.get_or_create(zalsa, || {
-                    // ingredient created by our jar is the struct ingredient.
+                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
-                    unsafe {
+                    })
-                        CACHE.get_or_create(zalsa, || {
-                            zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
-                        })
-                    }
                 }
 
-                pub fn ingredient_mut(zalsa_mut: &mut $zalsa::Zalsa) -> (&mut $zalsa_struct::IngredientImpl<Self>, &mut $zalsa::Runtime) {
+                pub fn ingredient_mut(db: &mut dyn $zalsa::Database) -> (&mut $zalsa_struct::IngredientImpl<Self>, &mut $zalsa::Runtime) {
+                    let zalsa_mut = db.zalsa_mut();
                     zalsa_mut.new_revision();
-                    let index = zalsa_mut.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>();
+                    let index = zalsa_mut.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create();
                     let (ingredient, runtime) = zalsa_mut.lookup_ingredient_mut(index);
                     let ingredient = ingredient.assert_type_mut::<$zalsa_struct::IngredientImpl<Self>>();
                     (ingredient, runtime)
@@ -170,8 +149,8 @@ macro_rules! setup_input_struct {
             impl $zalsa::SalsaStructInDb for $Struct {
                 type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
-                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
+                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
                 }
 
                 #[inline]
@@ -182,16 +161,6 @@ macro_rules! setup_input_struct {
                         $zalsa::None
                     }
                 }
-
-                #[inline]
-                unsafe fn memo_table(
-                    zalsa: &$zalsa::Zalsa,
-                    id: $zalsa::Id,
-                    current_revision: $zalsa::Revision,
-                ) -> $zalsa::MemoTableWithTypes<'_> {
-                    // SAFETY: Guaranteed by caller.
-                    unsafe { zalsa.table().memos::<$zalsa_struct::Value<$Configuration>>(id, current_revision) }
-                }
             }
 
             impl $Struct {
@@ -216,10 +185,8 @@ macro_rules! setup_input_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let (zalsa, zalsa_local) = db.zalsas();
+                        let fields = $Configuration::ingredient_(db.zalsa()).field(
-                        let fields = $Configuration::ingredient_(zalsa).field(
+                            db.as_dyn_database(),
-                            zalsa,
-                            zalsa_local,
                             self,
                             $field_index,
                         );
@@ -238,8 +205,7 @@ macro_rules! setup_input_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let zalsa = db.zalsa_mut();
+                        let (ingredient, revision) = $Configuration::ingredient_mut(db.as_dyn_database_mut());
-                        let (ingredient, revision) = $Configuration::ingredient_mut(zalsa);
                         $zalsa::input::SetterImpl::new(
                             revision,
                             self,
@@ -278,8 +244,7 @@ macro_rules! setup_input_struct {
                     $(for<'__trivial_bounds> $field_ty: std::fmt::Debug),*
                 {
                     $zalsa::with_attached_database(|db| {
-                        let zalsa = db.zalsa();
+                        let fields = $Configuration::ingredient(db).leak_fields(db, this);
-                        let fields = $Configuration::ingredient_(zalsa).leak_fields(zalsa, this);
                         let mut f = f.debug_struct(stringify!($Struct));
                         let f = f.field("[salsa id]", &$zalsa::AsId::as_id(&this));
                         $(
@@ -308,11 +273,11 @@ macro_rules! setup_input_struct {
                     // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                     $Db: ?Sized + salsa::Database
                 {
-                    let (zalsa, zalsa_local) = db.zalsas();
+                    let zalsa = db.zalsa();
                     let current_revision = zalsa.current_revision();
                     let ingredient = $Configuration::ingredient_(zalsa);
                     let (fields, revision, durabilities) = builder::builder_into_inner(self, current_revision);
-                    ingredient.new_input(zalsa, zalsa_local, fields, revision, durabilities)
+                    ingredient.new_input(db.as_dyn_database(), fields, revision, durabilities)
                 }
             }
 

components/salsa-macro-rules/src/setup_interned_struct.rs

@@ -66,9 +66,6 @@ macro_rules! setup_interned_struct {
         // If true, generate a debug impl.
         generate_debug_impl: $generate_debug_impl:tt,
 
-        // The function used to implement `C::heap_size`.
-        heap_size_fn: $($heap_size_fn:path)?,
-
         // Annoyingly macro-rules hygiene does not extend to items defined in the macro.
         // We have the procedural macro generate names for those items that are
         // not used elsewhere in the user's code.
@@ -84,7 +81,7 @@ macro_rules! setup_interned_struct {
         #[derive(Copy, Clone, PartialEq, Eq, Hash)]
         $vis struct $Struct< $($db_lt_arg)? >(
             $Id,
-            std::marker::PhantomData<fn() -> &$interior_lt ()>
+            std::marker::PhantomData < & $interior_lt salsa::plumbing::interned::Value <$StructWithStatic> >
         );
 
         #[allow(clippy::all)]
@@ -95,15 +92,6 @@ macro_rules! setup_interned_struct {
 
             type $Configuration = $StructWithStatic;
 
-            impl<$($db_lt_arg)?> $zalsa::HasJar for $Struct<$($db_lt_arg)?> {
-                type Jar = $zalsa_struct::JarImpl<$Configuration>;
-                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
-            }
-
-            $zalsa::register_jar! {
-                $zalsa::ErasedJar::erase::<$StructWithStatic>()
-            }
-
             type $StructDataIdent<$db_lt> = ($($field_ty,)*);
 
             /// Key to use during hash lookups. Each field is some type that implements `Lookup<T>`
@@ -149,27 +137,20 @@ macro_rules! setup_interned_struct {
                 )?
                 type Fields<'a> = $StructDataIdent<'a>;
                 type Struct<'db> = $Struct< $($db_lt_arg)? >;
-
-                $(
-                    fn heap_size(value: &Self::Fields<'_>) -> Option<usize> {
-                        Some($heap_size_fn(value))
-                    }
-                )?
             }
 
             impl $Configuration {
-                pub fn ingredient(zalsa: &$zalsa::Zalsa) -> &$zalsa_struct::IngredientImpl<Self>
+                pub fn ingredient<Db>(db: &Db) -> &$zalsa_struct::IngredientImpl<Self>
+                where
+                    Db: ?Sized + $zalsa::Database,
                 {
                     static CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Configuration>> =
                         $zalsa::IngredientCache::new();
 
-                    // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the only
+                    let zalsa = db.zalsa();
-                    // ingredient created by our jar is the struct ingredient.
+                    CACHE.get_or_create(zalsa, || {
-                    unsafe {
+                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
-                        CACHE.get_or_create(zalsa, || {
+                    })
-                            zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
-                        })
-                    }
                 }
             }
 
@@ -200,8 +181,8 @@ macro_rules! setup_interned_struct {
             impl< $($db_lt_arg)? > $zalsa::SalsaStructInDb for $Struct< $($db_lt_arg)? > {
                 type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
-                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
+                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
                 }
 
                 #[inline]
@@ -212,16 +193,6 @@ macro_rules! setup_interned_struct {
                         $zalsa::None
                     }
                 }
-
-                #[inline]
-                unsafe fn memo_table(
-                    zalsa: &$zalsa::Zalsa,
-                    id: $zalsa::Id,
-                    current_revision: $zalsa::Revision,
-                ) -> $zalsa::MemoTableWithTypes<'_> {
-                    // SAFETY: Guaranteed by caller.
-                    unsafe { zalsa.table().memos::<$zalsa_struct::Value<$Configuration>>(id, current_revision) }
-                }
             }
 
             unsafe impl< $($db_lt_arg)? > $zalsa::Update for $Struct< $($db_lt_arg)? > {
@@ -244,8 +215,7 @@ macro_rules! setup_interned_struct {
                         $field_ty: $zalsa::interned::HashEqLike<$indexed_ty>,
                     )*
                 {
-                    let (zalsa, zalsa_local) = db.zalsas();
+                    $Configuration::ingredient(db).intern(db.as_dyn_database(),
-                    $Configuration::ingredient(zalsa).intern(zalsa, zalsa_local,
                         StructKey::<$db_lt>($($field_id,)* std::marker::PhantomData::default()), |_, data| ($($zalsa::interned::Lookup::into_owned(data.$field_index),)*))
                 }
 
@@ -256,8 +226,7 @@ macro_rules! setup_interned_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let zalsa = db.zalsa();
+                        let fields = $Configuration::ingredient(db).fields(db.as_dyn_database(), self);
-                        let fields = $Configuration::ingredient(zalsa).fields(zalsa, self);
                         $zalsa::return_mode_expression!(
                             $field_option,
                             $field_ty,
@@ -269,8 +238,7 @@ macro_rules! setup_interned_struct {
                 /// Default debug formatting for this struct (may be useful if you define your own `Debug` impl)
                 pub fn default_debug_fmt(this: Self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                     $zalsa::with_attached_database(|db| {
-                        let zalsa = db.zalsa();
+                        let fields = $Configuration::ingredient(db).fields(db.as_dyn_database(), this);
-                        let fields = $Configuration::ingredient(zalsa).fields(zalsa, this);
                         let mut f = f.debug_struct(stringify!($Struct));
                         $(
                             let f = f.field(stringify!($field_id), &fields.$field_index);

components/salsa-macro-rules/src/setup_tracked_fn.rs

@@ -91,16 +91,6 @@ macro_rules! setup_tracked_fn {
 
             struct $Configuration;
 
-            $zalsa::register_jar! {
-                $zalsa::ErasedJar::erase::<$fn_name>()
-            }
-
-            #[allow(non_local_definitions)]
-            impl $zalsa::HasJar for $fn_name {
-                type Jar = $fn_name;
-                const KIND: $zalsa::JarKind = $zalsa::JarKind::TrackedFn;
-            }
-
             static $FN_CACHE: $zalsa::IngredientCache<$zalsa::function::IngredientImpl<$Configuration>> =
                 $zalsa::IngredientCache::new();
 
@@ -109,7 +99,7 @@ macro_rules! setup_tracked_fn {
                     #[derive(Copy, Clone)]
                     struct $InternedData<$db_lt>(
                         salsa::Id,
-                        std::marker::PhantomData<fn() -> &$db_lt ()>,
+                        std::marker::PhantomData<&$db_lt $zalsa::interned::Value<$Configuration>>,
                     );
 
                     static $INTERN_CACHE: $zalsa::IngredientCache<$zalsa::interned::IngredientImpl<$Configuration>> =
@@ -118,7 +108,7 @@ macro_rules! setup_tracked_fn {
                     impl $zalsa::SalsaStructInDb for $InternedData<'_> {
                         type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                        fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                        fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
                             $zalsa::IngredientIndices::empty()
                         }
 
@@ -130,16 +120,6 @@ macro_rules! setup_tracked_fn {
                                 None
                             }
                         }
-
-                        #[inline]
-                        unsafe fn memo_table(
-                            zalsa: &$zalsa::Zalsa,
-                            id: $zalsa::Id,
-                            current_revision: $zalsa::Revision,
-                        ) -> $zalsa::MemoTableWithTypes<'_> {
-                            // SAFETY: Guaranteed by caller.
-                            unsafe { zalsa.table().memos::<$zalsa::interned::Value<$Configuration>>(id, current_revision) }
-                        }
                     }
 
                     impl $zalsa::AsId for $InternedData<'_> {
@@ -175,27 +155,27 @@ macro_rules! setup_tracked_fn {
             impl $Configuration {
                 fn fn_ingredient(db: &dyn $Db) -> &$zalsa::function::IngredientImpl<$Configuration> {
                     let zalsa = db.zalsa();
-                    Self::fn_ingredient_(db, zalsa)
+                    $FN_CACHE.get_or_create(zalsa, || {
-                }
+                        let jar_entry = zalsa.lookup_jar_by_type::<$Configuration>();
 
-                #[inline]
+                        // If the ingredient has already been inserted, we know that the downcaster
-                fn fn_ingredient_<'z>(db: &dyn $Db, zalsa: &'z $zalsa::Zalsa) -> &'z $zalsa::function::IngredientImpl<$Configuration> {
+                        // has also been registered. This is a fast-path for multi-database use cases
-                    // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the first
+                        // that bypass the ingredient cache and will always execute this closure.
-                    // ingredient created by our jar is the function ingredient.
+                        if let Some(index) = jar_entry.get() {
-                    unsafe {
+                            return index;
-                        $FN_CACHE.get_or_create(zalsa, || zalsa.lookup_jar_by_type::<$fn_name>())
+                        }
-                    }
+
-                    .get_or_init(|| *<dyn $Db as $Db>::zalsa_register_downcaster(db))
+                        <dyn $Db as $Db>::zalsa_register_downcaster(db);
+                        jar_entry.get_or_create()
+                    })
                 }
 
                 pub fn fn_ingredient_mut(db: &mut dyn $Db) -> &mut $zalsa::function::IngredientImpl<Self> {
-                    let view = *<dyn $Db as $Db>::zalsa_register_downcaster(db);
+                    <dyn $Db as $Db>::zalsa_register_downcaster(db);
                     let zalsa_mut = db.zalsa_mut();
-                    let index = zalsa_mut.lookup_jar_by_type::<$fn_name>();
+                    let index = zalsa_mut.lookup_jar_by_type::<$Configuration>().get_or_create();
                     let (ingredient, _) = zalsa_mut.lookup_ingredient_mut(index);
-                    let ingredient = ingredient.assert_type_mut::<$zalsa::function::IngredientImpl<Self>>();
+                    ingredient.assert_type_mut::<$zalsa::function::IngredientImpl<Self>>()
-                    ingredient.get_or_init(|| view);
-                    ingredient
                 }
 
                 $zalsa::macro_if! { $needs_interner =>
@@ -203,19 +183,10 @@ macro_rules! setup_tracked_fn {
                         db: &dyn $Db,
                     ) -> &$zalsa::interned::IngredientImpl<$Configuration> {
                         let zalsa = db.zalsa();
-                        Self::intern_ingredient_(zalsa)
+                        $INTERN_CACHE.get_or_create(zalsa, || {
-                    }
+                            <dyn $Db as $Db>::zalsa_register_downcaster(db);
-                    #[inline]
+                            zalsa.lookup_jar_by_type::<$Configuration>().get_or_create().successor(0)
-                    fn intern_ingredient_<'z>(
+                        })
-                        zalsa: &'z $zalsa::Zalsa
-                    ) -> &'z $zalsa::interned::IngredientImpl<$Configuration> {
-                        // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the second
-                        // ingredient created by our jar is the interned ingredient (given `needs_interner`).
-                        unsafe {
-                            $INTERN_CACHE.get_or_create(zalsa, || {
-                                zalsa.lookup_jar_by_type::<$fn_name>().successor(0)
-                            })
-                        }
                     }
                 }
             }
@@ -240,8 +211,8 @@ macro_rules! setup_tracked_fn {
                 $($values_equal)+
 
                 $(
-                    fn heap_size(value: &Self::Output<'_>) -> Option<usize> {
+                    fn heap_size(value: &Self::Output<'_>) -> usize {
-                        Some($heap_size_fn(value))
+                        $heap_size_fn(value)
                     }
                 )?
 
@@ -266,42 +237,53 @@ macro_rules! setup_tracked_fn {
                     $($cycle_recovery_fn)*(db, value, count, $($input_id),*)
                 }
 
-                fn id_to_input<$db_lt>(zalsa: &$db_lt $zalsa::Zalsa, key: salsa::Id) -> Self::Input<$db_lt> {
+                fn id_to_input<$db_lt>(db: &$db_lt Self::DbView, key: salsa::Id) -> Self::Input<$db_lt> {
                     $zalsa::macro_if! {
                         if $needs_interner {
-                            $Configuration::intern_ingredient_(zalsa).data(zalsa, key).clone()
+                            $Configuration::intern_ingredient(db).data(db.as_dyn_database(), key).clone()
                         } else {
-                            $zalsa::FromIdWithDb::from_id(key, zalsa)
+                            $zalsa::FromIdWithDb::from_id(key, db.zalsa())
                         }
                     }
                 }
             }
 
-            #[allow(non_local_definitions)]
+            impl $zalsa::Jar for $Configuration {
-            impl $zalsa::Jar for $fn_name {
+                fn create_dependencies(zalsa: &$zalsa::Zalsa) -> $zalsa::IngredientIndices
+                where
+                    Self: Sized
+                {
+                    $zalsa::macro_if! {
+                        if $needs_interner {
+                            $zalsa::IngredientIndices::empty()
+                        } else {
+                            <$InternedData as $zalsa::SalsaStructInDb>::lookup_or_create_ingredient_index(zalsa)
+                        }
+                    }
+                }
+
                 fn create_ingredients(
-                    zalsa: &mut $zalsa::Zalsa,
+                    zalsa: &$zalsa::Zalsa,
                     first_index: $zalsa::IngredientIndex,
+                    struct_index: $zalsa::IngredientIndices,
                 ) -> Vec<Box<dyn $zalsa::Ingredient>> {
                     let struct_index: $zalsa::IngredientIndices = $zalsa::macro_if! {
                         if $needs_interner {
                             first_index.successor(0).into()
                         } else {
-                            // Note that struct ingredients are created before tracked functions,
+                            struct_index
-                            // so this cannot panic.
-                            <$InternedData as $zalsa::SalsaStructInDb>::lookup_ingredient_index(zalsa)
                         }
                     };
 
                     $zalsa::macro_if! { $needs_interner =>
-                        let mut intern_ingredient = <$zalsa::interned::IngredientImpl<$Configuration>>::new(
+                        let intern_ingredient = <$zalsa::interned::IngredientImpl<$Configuration>>::new(
                             first_index.successor(0)
                         );
                     }
 
                     let intern_ingredient_memo_types = $zalsa::macro_if! {
                         if $needs_interner {
-                            Some($zalsa::Ingredient::memo_table_types_mut(&mut intern_ingredient))
+                            Some($zalsa::Ingredient::memo_table_types(&intern_ingredient))
                         } else {
                             None
                         }
@@ -321,6 +303,7 @@ macro_rules! setup_tracked_fn {
                         first_index,
                         memo_ingredient_indices,
                         $lru,
+                        zalsa.views().downcaster_for::<dyn $Db>(),
                     );
                     $zalsa::macro_if! {
                         if $needs_interner {
@@ -343,23 +326,20 @@ macro_rules! setup_tracked_fn {
 
             #[allow(non_local_definitions)]
             impl $fn_name {
-                $zalsa::gate_accumulated! {
+                pub fn accumulated<$db_lt, A: salsa::Accumulator>(
-                    pub fn accumulated<$db_lt, A: salsa::Accumulator>(
+                    $db: &$db_lt dyn $Db,
-                        $db: &$db_lt dyn $Db,
+                    $($input_id: $interned_input_ty,)*
-                        $($input_id: $interned_input_ty,)*
+                ) -> Vec<&$db_lt A> {
-                    ) -> Vec<&$db_lt A> {
+                    use salsa::plumbing as $zalsa;
-                        use salsa::plumbing as $zalsa;
+                    let key = $zalsa::macro_if! {
-                        let key = $zalsa::macro_if! {
+                        if $needs_interner {
-                            if $needs_interner {{
+                            $Configuration::intern_ingredient($db).intern_id($db.as_dyn_database(), ($($input_id),*), |_, data| data)
-                                let (zalsa, zalsa_local) = $db.zalsas();
+                        } else {
-                                $Configuration::intern_ingredient($db).intern_id(zalsa, zalsa_local, ($($input_id),*), |_, data| data)
+                            $zalsa::AsId::as_id(&($($input_id),*))
-                            }} else {
+                        }
-                                $zalsa::AsId::as_id(&($($input_id),*))
+                    };
-                            }
-                        };
 
-                        $Configuration::fn_ingredient($db).accumulated_by::<A>($db, key)
+                    $Configuration::fn_ingredient($db).accumulated_by::<A>($db, key)
-                    }
                 }
 
                 $zalsa::macro_if! { $is_specifiable =>
@@ -392,24 +372,20 @@ macro_rules! setup_tracked_fn {
             }
 
             $zalsa::attach($db, || {
-                let (zalsa, zalsa_local) = $db.zalsas();
                 let result = $zalsa::macro_if! {
                     if $needs_interner {
                         {
-                            let key = $Configuration::intern_ingredient_(zalsa).intern_id(zalsa, zalsa_local, ($($input_id),*), |_, data| data);
+                            let key = $Configuration::intern_ingredient($db).intern_id($db.as_dyn_database(), ($($input_id),*), |_, data| data);
-                            $Configuration::fn_ingredient_($db, zalsa).fetch($db, zalsa, zalsa_local, key)
+                            $Configuration::fn_ingredient($db).fetch($db, key)
                         }
                     } else {
-                        {
+                        $Configuration::fn_ingredient($db).fetch($db, $zalsa::AsId::as_id(&($($input_id),*)))
-                            $Configuration::fn_ingredient_($db, zalsa).fetch($db, zalsa, zalsa_local, $zalsa::AsId::as_id(&($($input_id),*)))
-                        }
                     }
                 };
 
                 $zalsa::return_mode_expression!(($return_mode, __, __), $output_ty, result,)
             })
         }
-
         // The struct needs be last in the macro expansion in order to make the tracked
         // function's ident be identified as a function, not a struct, during semantic highlighting.
         // for more details, see https://github.com/salsa-rs/salsa/pull/612.

components/salsa-macro-rules/src/setup_tracked_struct.rs

@@ -88,9 +88,6 @@ macro_rules! setup_tracked_struct {
         // If true, generate a debug impl.
         generate_debug_impl: $generate_debug_impl:tt,
 
-        // The function used to implement `C::heap_size`.
-        heap_size_fn: $($heap_size_fn:path)?,
-
         // Annoyingly macro-rules hygiene does not extend to items defined in the macro.
         // We have the procedural macro generate names for those items that are
         // not used elsewhere in the user's code.
@@ -107,11 +104,11 @@ macro_rules! setup_tracked_struct {
         #[derive(Copy, Clone, PartialEq, Eq, Hash)]
         $vis struct $Struct<$db_lt>(
             salsa::Id,
-            std::marker::PhantomData<fn() -> &$db_lt ()>
+            std::marker::PhantomData < & $db_lt salsa::plumbing::tracked_struct::Value < $Struct<'static> > >
         );
 
-        #[allow(dead_code)]
         #[allow(clippy::all)]
+        #[allow(dead_code)]
         const _: () = {
             use salsa::plumbing as $zalsa;
             use $zalsa::tracked_struct as $zalsa_struct;
@@ -119,15 +116,6 @@ macro_rules! setup_tracked_struct {
 
             type $Configuration = $Struct<'static>;
 
-            impl<$db_lt> $zalsa::HasJar for $Struct<$db_lt> {
-                type Jar = $zalsa_struct::JarImpl<$Configuration>;
-                const KIND: $zalsa::JarKind = $zalsa::JarKind::Struct;
-            }
-
-            $zalsa::register_jar! {
-                $zalsa::ErasedJar::erase::<$Struct<'static>>()
-            }
-
             impl $zalsa_struct::Configuration for $Configuration {
                 const LOCATION: $zalsa::Location = $zalsa::Location {
                     file: file!(),
@@ -188,12 +176,6 @@ macro_rules! setup_tracked_struct {
                         )* false
                     }
                 }
-
-                $(
-                    fn heap_size(value: &Self::Fields<'_>) -> Option<usize> {
-                        Some($heap_size_fn(value))
-                    }
-                )?
             }
 
             impl $Configuration {
@@ -205,13 +187,9 @@ macro_rules! setup_tracked_struct {
                     static CACHE: $zalsa::IngredientCache<$zalsa_struct::IngredientImpl<$Configuration>> =
                         $zalsa::IngredientCache::new();
 
-                    // SAFETY: `lookup_jar_by_type` returns a valid ingredient index, and the only
+                    CACHE.get_or_create(zalsa, || {
-                    // ingredient created by our jar is the struct ingredient.
+                        zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create()
-                    unsafe {
+                    })
-                        CACHE.get_or_create(zalsa, || {
-                            zalsa.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>()
-                        })
-                    }
                 }
             }
 
@@ -232,8 +210,8 @@ macro_rules! setup_tracked_struct {
             impl $zalsa::SalsaStructInDb for $Struct<'_> {
                 type MemoIngredientMap = $zalsa::MemoIngredientSingletonIndex;
 
-                fn lookup_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
+                fn lookup_or_create_ingredient_index(aux: &$zalsa::Zalsa) -> $zalsa::IngredientIndices {
-                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().into()
+                    aux.lookup_jar_by_type::<$zalsa_struct::JarImpl<$Configuration>>().get_or_create().into()
                 }
 
                 #[inline]
@@ -244,16 +222,6 @@ macro_rules! setup_tracked_struct {
                         $zalsa::None
                     }
                 }
-
-                #[inline]
-                unsafe fn memo_table(
-                    zalsa: &$zalsa::Zalsa,
-                    id: $zalsa::Id,
-                    current_revision: $zalsa::Revision,
-                ) -> $zalsa::MemoTableWithTypes<'_> {
-                    // SAFETY: Guaranteed by caller.
-                    unsafe { zalsa.table().memos::<$zalsa_struct::Value<$Configuration>>(id, current_revision) }
-                }
             }
 
             impl $zalsa::TrackedStructInDb for $Struct<'_> {
@@ -291,9 +259,8 @@ macro_rules! setup_tracked_struct {
                     // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                     $Db: ?Sized + $zalsa::Database,
                 {
-                    let (zalsa, zalsa_local) = db.zalsas();
+                    $Configuration::ingredient(db.as_dyn_database()).new_struct(
-                    $Configuration::ingredient_(zalsa).new_struct(
+                        db.as_dyn_database(),
-                        zalsa,zalsa_local,
                         ($($field_id,)*)
                     )
                 }
@@ -305,8 +272,8 @@ macro_rules! setup_tracked_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let (zalsa, zalsa_local) = db.zalsas();
+                        let db = db.as_dyn_database();
-                        let fields = $Configuration::ingredient_(zalsa).tracked_field(zalsa, zalsa_local, self, $relative_tracked_index);
+                        let fields = $Configuration::ingredient(db).tracked_field(db, self, $relative_tracked_index);
                         $crate::return_mode_expression!(
                             $tracked_option,
                             $tracked_ty,
@@ -322,8 +289,8 @@ macro_rules! setup_tracked_struct {
                         // FIXME(rust-lang/rust#65991): The `db` argument *should* have the type `dyn Database`
                         $Db: ?Sized + $zalsa::Database,
                     {
-                        let zalsa = db.zalsa();
+                        let db = db.as_dyn_database();
-                        let fields = $Configuration::ingredient_(zalsa).untracked_field(zalsa, self);
+                        let fields = $Configuration::ingredient(db).untracked_field(db, self);
                         $crate::return_mode_expression!(
                             $untracked_option,
                             $untracked_ty,
@@ -345,8 +312,7 @@ macro_rules! setup_tracked_struct {
                     $(for<$db_lt> $field_ty: std::fmt::Debug),*
                 {
                     $zalsa::with_attached_database(|db| {
-                        let zalsa = db.zalsa();
+                        let fields = $Configuration::ingredient(db).leak_fields(db, this);
-                        let fields = $Configuration::ingredient_(zalsa).leak_fields(zalsa, this);
                         let mut f = f.debug_struct(stringify!($Struct));
                         let f = f.field("[salsa id]", &$zalsa::AsId::as_id(&this));
                         $(

components/salsa-macros/Cargo.toml

@@ -14,5 +14,5 @@ proc-macro = true
 [dependencies]
 proc-macro2 = "1.0"
 quote = "1.0"
-syn = { version = "2.0.104", features = ["full", "visit-mut"] }
+syn = { version = "2.0.101", features = ["full", "visit-mut"] }
 synstructure = "0.13.2"

components/salsa-macros/src/db.rs

@@ -110,14 +110,18 @@ impl DbMacro {
         let trait_name = &input.ident;
         input.items.push(parse_quote! {
             #[doc(hidden)]
-            fn zalsa_register_downcaster(&self) -> &salsa::plumbing::DatabaseDownCaster<dyn #trait_name>;
+            fn zalsa_register_downcaster(&self);
         });
 
-        let comment = format!(" downcast `Self` to a [`dyn {trait_name}`]");
+        let comment = format!(" Downcast a [`dyn Database`] to a [`dyn {trait_name}`]");
         input.items.push(parse_quote! {
             #[doc = #comment]
+            ///
+            /// # Safety
+            ///
+            /// The input database must be of type `Self`.
             #[doc(hidden)]
-            fn downcast(&self) -> &dyn #trait_name where Self: Sized;
+            unsafe fn downcast(db: &dyn salsa::plumbing::Database) -> &dyn #trait_name where Self: Sized;
         });
         Ok(())
     }
@@ -131,20 +135,19 @@ impl DbMacro {
         };
 
         input.items.push(parse_quote! {
-            #[cold]
-            #[inline(never)]
             #[doc(hidden)]
-            fn zalsa_register_downcaster(&self) -> &salsa::plumbing::DatabaseDownCaster<dyn #TraitPath> {
+            #[inline(always)]
-                salsa::plumbing::views(self).add::<Self, dyn #TraitPath>(unsafe {
+            fn zalsa_register_downcaster(&self)  {
-                    ::std::mem::transmute(<Self as #TraitPath>::downcast as fn(_) -> _)
+                salsa::plumbing::views(self).add(<Self as #TraitPath>::downcast);
-                })
             }
         });
         input.items.push(parse_quote! {
             #[doc(hidden)]
             #[inline(always)]
-            fn downcast(&self) -> &dyn #TraitPath where Self: Sized {
+            unsafe fn downcast(db: &dyn salsa::plumbing::Database) -> &dyn #TraitPath where Self: Sized {
-                self
+                debug_assert_eq!(db.type_id(), ::core::any::TypeId::of::<Self>());
+                // SAFETY: The input database must be of type `Self`.
+                unsafe { &*salsa::plumbing::transmute_data_ptr::<dyn salsa::plumbing::Database, Self>(db) }
             }
         });
         Ok(())

components/salsa-macros/src/fn_util.rs

@@ -15,7 +15,7 @@ pub fn input_ids(hygiene: &Hygiene, sig: &syn::Signature, skip: usize) -> Vec<sy
                 }
             }
 
-            hygiene.ident(format!("input{index}"))
+            hygiene.ident(&format!("input{index}"))
         })
         .collect()
 }

components/salsa-macros/src/hygiene.rs

@@ -50,10 +50,10 @@ impl Hygiene {
     /// Generates an identifier similar to `text` but
     /// distinct from any identifiers that appear in the user's
     /// code.
-    pub(crate) fn ident(&self, text: impl AsRef<str>) -> syn::Ident {
+    pub(crate) fn ident(&self, text: &str) -> syn::Ident {
         // Make the default be `foo_` rather than `foo` -- this helps detect
         // cases where people wrote `foo` instead of `#foo` or `$foo` in the generated code.
-        let mut buffer = format!("{}_", text.as_ref());
+        let mut buffer = format!("{text}_");
 
         while self.user_tokens.contains(&buffer) {
             buffer.push('_');
@@ -61,12 +61,4 @@ impl Hygiene {
 
         syn::Ident::new(&buffer, proc_macro2::Span::call_site())
     }
-
-    /// Generates an identifier similar to `text` but distinct from any identifiers
-    /// that appear in the user's code.
-    ///
-    /// The identifier must be unique relative to the `scope` identifier.
-    pub(crate) fn scoped_ident(&self, scope: &syn::Ident, text: &str) -> syn::Ident {
-        self.ident(format!("{scope}_{text}"))
-    }
 }

components/salsa-macros/src/input.rs

@@ -65,7 +65,7 @@ impl crate::options::AllowedOptions for InputStruct {
 
     const REVISIONS: bool = false;
 
-    const HEAP_SIZE: bool = true;
+    const HEAP_SIZE: bool = false;
 
     const SELF_TY: bool = false;
 }
@@ -112,7 +112,6 @@ impl Macro {
         let field_attrs = salsa_struct.field_attrs();
         let is_singleton = self.args.singleton.is_some();
         let generate_debug_impl = salsa_struct.generate_debug_impl();
-        let heap_size_fn = self.args.heap_size_fn.iter();
 
         let zalsa = self.hygiene.ident("zalsa");
         let zalsa_struct = self.hygiene.ident("zalsa_struct");
@@ -141,7 +140,6 @@ impl Macro {
                     num_fields: #num_fields,
                     is_singleton: #is_singleton,
                     generate_debug_impl: #generate_debug_impl,
-                    heap_size_fn: #(#heap_size_fn)*,
                     unused_names: [
                         #zalsa,
                         #zalsa_struct,

components/salsa-macros/src/interned.rs

@@ -65,7 +65,7 @@ impl crate::options::AllowedOptions for InternedStruct {
 
     const REVISIONS: bool = true;
 
-    const HEAP_SIZE: bool = true;
+    const HEAP_SIZE: bool = false;
 
     const SELF_TY: bool = false;
 }
@@ -131,8 +131,6 @@ impl Macro {
             (None, quote!(#struct_ident), static_lifetime)
         };
 
-        let heap_size_fn = self.args.heap_size_fn.iter();
-
         let zalsa = self.hygiene.ident("zalsa");
         let zalsa_struct = self.hygiene.ident("zalsa_struct");
         let Configuration = self.hygiene.ident("Configuration");
@@ -163,7 +161,6 @@ impl Macro {
                     field_attrs: [#([#(#field_unused_attrs),*]),*],
                     num_fields: #num_fields,
                     generate_debug_impl: #generate_debug_impl,
-                    heap_size_fn: #(#heap_size_fn)*,
                     unused_names: [
                         #zalsa,
                         #zalsa_struct,

components/salsa-macros/src/options.rs

@@ -527,7 +527,7 @@ impl<A: AllowedOptions> quote::ToTokens for Options<A> {
             tokens.extend(quote::quote! { revisions = #revisions, });
         }
         if let Some(heap_size_fn) = heap_size_fn {
-            tokens.extend(quote::quote! { heap_size = #heap_size_fn, });
+            tokens.extend(quote::quote! { heap_size_fn = #heap_size_fn, });
         }
         if let Some(self_ty) = self_ty {
             tokens.extend(quote::quote! { self_ty = #self_ty, });

components/salsa-macros/src/supertype.rs

@@ -72,8 +72,8 @@ fn enum_impl(enum_item: syn::ItemEnum) -> syn::Result<TokenStream> {
             type MemoIngredientMap = zalsa::MemoIngredientIndices;
 
             #[inline]
-            fn lookup_ingredient_index(__zalsa: &zalsa::Zalsa) -> zalsa::IngredientIndices {
+            fn lookup_or_create_ingredient_index(__zalsa: &zalsa::Zalsa) -> zalsa::IngredientIndices {
-                zalsa::IngredientIndices::merge([ #( <#variant_types as zalsa::SalsaStructInDb>::lookup_ingredient_index(__zalsa) ),* ])
+                zalsa::IngredientIndices::merge([ #( <#variant_types as zalsa::SalsaStructInDb>::lookup_or_create_ingredient_index(__zalsa) ),* ])
             }
 
             #[inline]
@@ -89,19 +89,6 @@ fn enum_impl(enum_item: syn::ItemEnum) -> syn::Result<TokenStream> {
                     None
                 }
             }
-
-            #[inline]
-            unsafe fn memo_table(
-                zalsa: &zalsa::Zalsa,
-                id: zalsa::Id,
-                current_revision: zalsa::Revision,
-            ) -> zalsa::MemoTableWithTypes<'_> {
-                // Note that we need to use `dyn_memos` here, as the `Id` could map to any variant
-                // of the supertype enum.
-                //
-                // SAFETY: Guaranteed by caller.
-                unsafe { zalsa.table().dyn_memos(id, current_revision) }
-            }
         }
     };
 

components/salsa-macros/src/tracked_fn.rs

@@ -132,10 +132,10 @@ impl Macro {
         inner_fn.sig.ident = self.hygiene.ident("inner");
 
         let zalsa = self.hygiene.ident("zalsa");
-        let Configuration = self.hygiene.scoped_ident(fn_name, "Configuration");
+        let Configuration = self.hygiene.ident("Configuration");
-        let InternedData = self.hygiene.scoped_ident(fn_name, "InternedData");
+        let InternedData = self.hygiene.ident("InternedData");
-        let FN_CACHE = self.hygiene.scoped_ident(fn_name, "FN_CACHE");
+        let FN_CACHE = self.hygiene.ident("FN_CACHE");
-        let INTERN_CACHE = self.hygiene.scoped_ident(fn_name, "INTERN_CACHE");
+        let INTERN_CACHE = self.hygiene.ident("INTERN_CACHE");
         let inner = &inner_fn.sig.ident;
 
         let function_type = function_type(&item);

components/salsa-macros/src/tracked_impl.rs

@@ -99,7 +99,7 @@ impl Macro {
         });
 
         let InnerTrait = self.hygiene.ident("InnerTrait");
-        let inner_fn_name = self.hygiene.ident(fn_item.sig.ident.to_string());
+        let inner_fn_name = self.hygiene.ident(&fn_item.sig.ident.to_string());
 
         let AssociatedFunctionArguments {
             self_token,

components/salsa-macros/src/tracked_struct.rs

@@ -61,7 +61,7 @@ impl crate::options::AllowedOptions for TrackedStruct {
 
     const REVISIONS: bool = false;
 
-    const HEAP_SIZE: bool = true;
+    const HEAP_SIZE: bool = false;
 
     const SELF_TY: bool = false;
 }
@@ -141,8 +141,6 @@ impl Macro {
             }
         });
 
-        let heap_size_fn = self.args.heap_size_fn.iter();
-
         let num_tracked_fields = salsa_struct.num_tracked_fields();
         let generate_debug_impl = salsa_struct.generate_debug_impl();
 
@@ -190,9 +188,6 @@ impl Macro {
 
                     num_tracked_fields: #num_tracked_fields,
                     generate_debug_impl: #generate_debug_impl,
-
-                    heap_size_fn: #(#heap_size_fn)*,
-
                     unused_names: [
                         #zalsa,
                         #zalsa_struct,

src/accumulator.rs

@@ -7,10 +7,10 @@ use std::panic::UnwindSafe;
 
 use accumulated::{Accumulated, AnyAccumulated};
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::function::VerifyResult;
 use crate::ingredient::{Ingredient, Jar};
-use crate::plumbing::ZalsaLocal;
+use crate::plumbing::{IngredientIndices, ZalsaLocal};
 use crate::sync::Arc;
 use crate::table::memo::MemoTableTypes;
 use crate::zalsa::{IngredientIndex, Zalsa};
@@ -44,8 +44,9 @@ impl<A: Accumulator> Default for JarImpl<A> {
 
 impl<A: Accumulator> Jar for JarImpl<A> {
     fn create_ingredients(
-        _zalsa: &mut Zalsa,
+        _zalsa: &Zalsa,
         first_index: IngredientIndex,
+        _dependencies: IngredientIndices,
     ) -> Vec<Box<dyn Ingredient>> {
         vec![Box::new(<IngredientImpl<A>>::new(first_index))]
     }
@@ -63,7 +64,7 @@ pub struct IngredientImpl<A: Accumulator> {
 impl<A: Accumulator> IngredientImpl<A> {
     /// Find the accumulator ingredient for `A` in the database, if any.
     pub fn from_zalsa(zalsa: &Zalsa) -> Option<&Self> {
-        let index = zalsa.lookup_jar_by_type::<JarImpl<A>>();
+        let index = zalsa.lookup_jar_by_type::<JarImpl<A>>().get_or_create();
         let ingredient = zalsa.lookup_ingredient(index).assert_type::<Self>();
         Some(ingredient)
     }
@@ -102,11 +103,10 @@ impl<A: Accumulator> Ingredient for IngredientImpl<A> {
 
     unsafe fn maybe_changed_after(
         &self,
-        _zalsa: &crate::zalsa::Zalsa,
+        _db: &dyn Database,
-        _db: crate::database::RawDatabase<'_>,
         _input: Id,
         _revision: Revision,
-        _cycle_heads: &mut CycleHeadKeys,
+        _cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
         panic!("nothing should ever depend on an accumulator directly")
     }
@@ -115,11 +115,7 @@ impl<A: Accumulator> Ingredient for IngredientImpl<A> {
         A::DEBUG_NAME
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        unreachable!("accumulator does not allocate pages")
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
         unreachable!("accumulator does not allocate pages")
     }
 }

src/active_query.rs

@@ -1,9 +1,7 @@
 use std::{fmt, mem, ops};
 
-#[cfg(feature = "accumulator")]
+use crate::accumulator::accumulated_map::{
-use crate::accumulator::{
+    AccumulatedMap, AtomicInputAccumulatedValues, InputAccumulatedValues,
-    accumulated_map::{AccumulatedMap, AtomicInputAccumulatedValues, InputAccumulatedValues},
-    Accumulator,
 };
 use crate::cycle::{CycleHeads, IterationCount};
 use crate::durability::Durability;
@@ -13,7 +11,7 @@ use crate::runtime::Stamp;
 use crate::sync::atomic::AtomicBool;
 use crate::tracked_struct::{Disambiguator, DisambiguatorMap, IdentityHash, IdentityMap};
 use crate::zalsa_local::{QueryEdge, QueryOrigin, QueryRevisions, QueryRevisionsExtra};
-use crate::Revision;
+use crate::{Accumulator, IngredientIndex, Revision};
 
 #[derive(Debug)]
 pub(crate) struct ActiveQuery {
@@ -53,12 +51,10 @@ pub(crate) struct ActiveQuery {
 
     /// Stores the values accumulated to the given ingredient.
     /// The type of accumulated value is erased but known to the ingredient.
-    #[cfg(feature = "accumulator")]
     accumulated: AccumulatedMap,
 
     /// [`InputAccumulatedValues::Empty`] if any input read during the query's execution
     /// has any accumulated values.
-    #[cfg(feature = "accumulator")]
     accumulated_inputs: InputAccumulatedValues,
 
     /// Provisional cycle results that this query depends on.
@@ -77,7 +73,7 @@ impl ActiveQuery {
         untracked_read: bool,
     ) {
         assert!(self.input_outputs.is_empty());
-        self.input_outputs.extend(edges.iter().cloned());
+        self.input_outputs = edges.iter().cloned().collect();
         self.durability = self.durability.min(durability);
         self.changed_at = self.changed_at.max(changed_at);
         self.untracked_read |= untracked_read;
@@ -88,21 +84,18 @@ impl ActiveQuery {
         input: DatabaseKeyIndex,
         durability: Durability,
         changed_at: Revision,
+        has_accumulated: bool,
+        accumulated_inputs: &AtomicInputAccumulatedValues,
         cycle_heads: &CycleHeads,
-        #[cfg(feature = "accumulator")] has_accumulated: bool,
-        #[cfg(feature = "accumulator")] accumulated_inputs: &AtomicInputAccumulatedValues,
     ) {
         self.durability = self.durability.min(durability);
         self.changed_at = self.changed_at.max(changed_at);
         self.input_outputs.insert(QueryEdge::input(input));
+        self.accumulated_inputs = self.accumulated_inputs.or_else(|| match has_accumulated {
+            true => InputAccumulatedValues::Any,
+            false => accumulated_inputs.load(),
+        });
         self.cycle_heads.extend(cycle_heads);
-        #[cfg(feature = "accumulator")]
-        {
-            self.accumulated_inputs = self.accumulated_inputs.or_else(|| match has_accumulated {
-                true => InputAccumulatedValues::Any,
-                false => accumulated_inputs.load(),
-            });
-        }
     }
 
     pub(super) fn add_read_simple(
@@ -128,8 +121,7 @@ impl ActiveQuery {
         self.changed_at = self.changed_at.max(revision);
     }
 
-    #[cfg(feature = "accumulator")]
+    pub(super) fn accumulate(&mut self, index: IngredientIndex, value: impl Accumulator) {
-    pub(super) fn accumulate(&mut self, index: crate::IngredientIndex, value: impl Accumulator) {
         self.accumulated.accumulate(index, value);
     }
 
@@ -177,12 +169,10 @@ impl ActiveQuery {
             untracked_read: false,
             disambiguator_map: Default::default(),
             tracked_struct_ids: Default::default(),
+            accumulated: Default::default(),
+            accumulated_inputs: Default::default(),
             cycle_heads: Default::default(),
             iteration_count,
-            #[cfg(feature = "accumulator")]
-            accumulated: Default::default(),
-            #[cfg(feature = "accumulator")]
-            accumulated_inputs: Default::default(),
         }
     }
 
@@ -195,12 +185,10 @@ impl ActiveQuery {
             untracked_read,
             ref mut disambiguator_map,
             ref mut tracked_struct_ids,
+            ref mut accumulated,
+            accumulated_inputs,
             ref mut cycle_heads,
             iteration_count,
-            #[cfg(feature = "accumulator")]
-            ref mut accumulated,
-            #[cfg(feature = "accumulator")]
-            accumulated_inputs,
         } = self;
 
         let origin = if untracked_read {
@@ -210,22 +198,19 @@ impl ActiveQuery {
         };
         disambiguator_map.clear();
 
-        #[cfg(feature = "accumulator")]
-        let accumulated_inputs = AtomicInputAccumulatedValues::new(accumulated_inputs);
         let verified_final = cycle_heads.is_empty();
         let extra = QueryRevisionsExtra::new(
-            #[cfg(feature = "accumulator")]
             mem::take(accumulated),
             mem::take(tracked_struct_ids),
             mem::take(cycle_heads),
             iteration_count,
         );
+        let accumulated_inputs = AtomicInputAccumulatedValues::new(accumulated_inputs);
 
         QueryRevisions {
             changed_at,
             durability,
             origin,
-            #[cfg(feature = "accumulator")]
             accumulated_inputs,
             verified_final: AtomicBool::new(verified_final),
             extra,
@@ -241,20 +226,17 @@ impl ActiveQuery {
             untracked_read: _,
             disambiguator_map,
             tracked_struct_ids,
+            accumulated,
+            accumulated_inputs: _,
             cycle_heads,
             iteration_count,
-            #[cfg(feature = "accumulator")]
-            accumulated,
-            #[cfg(feature = "accumulator")]
-                accumulated_inputs: _,
         } = self;
         input_outputs.clear();
         disambiguator_map.clear();
         tracked_struct_ids.clear();
+        accumulated.clear();
         *cycle_heads = Default::default();
         *iteration_count = IterationCount::initial();
-        #[cfg(feature = "accumulator")]
-        accumulated.clear();
     }
 
     fn reset_for(
@@ -270,17 +252,16 @@ impl ActiveQuery {
             untracked_read,
             disambiguator_map,
             tracked_struct_ids,
+            accumulated,
+            accumulated_inputs,
             cycle_heads,
             iteration_count,
-            #[cfg(feature = "accumulator")]
-            accumulated,
-            #[cfg(feature = "accumulator")]
-            accumulated_inputs,
         } = self;
         *database_key_index = new_database_key_index;
         *durability = Durability::MAX;
         *changed_at = Revision::start();
         *untracked_read = false;
+        *accumulated_inputs = Default::default();
         *iteration_count = new_iteration_count;
         debug_assert!(
             input_outputs.is_empty(),
@@ -298,14 +279,10 @@ impl ActiveQuery {
             cycle_heads.is_empty(),
             "`ActiveQuery::clear` or `ActiveQuery::into_revisions` should've been called"
         );
-        #[cfg(feature = "accumulator")]
+        debug_assert!(
-        {
+            accumulated.is_empty(),
-            *accumulated_inputs = Default::default();
+            "`ActiveQuery::clear` or `ActiveQuery::into_revisions` should've been called"
-            debug_assert!(
+        );
-                accumulated.is_empty(),
-                "`ActiveQuery::clear` or `ActiveQuery::into_revisions` should've been called"
-            );
-        }
     }
 }
 

src/cycle.rs

@@ -178,6 +178,22 @@ impl CycleHeads {
         }
     }
 
+    #[inline]
+    pub(crate) fn push_initial(&mut self, database_key_index: DatabaseKeyIndex) {
+        if let Some(existing) = self
+            .0
+            .iter()
+            .find(|candidate| candidate.database_key_index == database_key_index)
+        {
+            assert_eq!(existing.iteration_count, IterationCount::initial());
+        } else {
+            self.0.push(CycleHead {
+                database_key_index,
+                iteration_count: IterationCount::initial(),
+            });
+        }
+    }
+
     #[inline]
     pub(crate) fn extend(&mut self, other: &Self) {
         self.0.reserve(other.0.len());
@@ -231,37 +247,6 @@ pub(crate) fn empty_cycle_heads() -> &'static CycleHeads {
     EMPTY_CYCLE_HEADS.get_or_init(|| CycleHeads(ThinVec::new()))
 }
 
-/// Set of cycle head database keys.
-///
-/// Unlike [`CycleHeads`], this type doesn't track the iteration count
-/// of each cycle head.
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct CycleHeadKeys(Vec<DatabaseKeyIndex>);
-
-impl CycleHeadKeys {
-    pub(crate) fn new() -> Self {
-        Self(Vec::new())
-    }
-
-    pub(crate) fn insert(&mut self, database_key_index: DatabaseKeyIndex) {
-        if !self.0.contains(&database_key_index) {
-            self.0.push(database_key_index);
-        }
-    }
-
-    pub(crate) fn remove(&mut self, value: &DatabaseKeyIndex) -> bool {
-        let found = self.0.iter().position(|&head| head == *value);
-        let Some(found) = found else { return false };
-
-        self.0.swap_remove(found);
-        true
-    }
-
-    pub(crate) fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-}
-
 #[derive(Debug, PartialEq, Eq)]
 pub enum ProvisionalStatus {
     Provisional { iteration: IterationCount },

src/database.rs

@@ -1,39 +1,12 @@
+use std::any::Any;
 use std::borrow::Cow;
-use std::ptr::NonNull;
 
-use crate::views::DatabaseDownCaster;
 use crate::zalsa::{IngredientIndex, ZalsaDatabase};
 use crate::{Durability, Revision};
 
-#[derive(Copy, Clone)]
-pub struct RawDatabase<'db> {
-    pub(crate) ptr: NonNull<()>,
-    _marker: std::marker::PhantomData<&'db dyn Database>,
-}
-
-impl<'db, Db: Database + ?Sized> From<&'db Db> for RawDatabase<'db> {
-    #[inline]
-    fn from(db: &'db Db) -> Self {
-        RawDatabase {
-            ptr: NonNull::from(db).cast(),
-            _marker: std::marker::PhantomData,
-        }
-    }
-}
-
-impl<'db, Db: Database + ?Sized> From<&'db mut Db> for RawDatabase<'db> {
-    #[inline]
-    fn from(db: &'db mut Db) -> Self {
-        RawDatabase {
-            ptr: NonNull::from(db).cast(),
-            _marker: std::marker::PhantomData,
-        }
-    }
-}
-
 /// The trait implemented by all Salsa databases.
 /// You can create your own subtraits of this trait using the `#[salsa::db]`(`crate::db`) procedural macro.
-pub trait Database: Send + ZalsaDatabase + AsDynDatabase {
+pub trait Database: Send + AsDynDatabase + Any + ZalsaDatabase {
     /// Enforces current LRU limits, evicting entries if necessary.
     ///
     /// **WARNING:** Just like an ordinary write, this method triggers
@@ -59,14 +32,6 @@ pub trait Database: Send + ZalsaDatabase + AsDynDatabase {
         zalsa_mut.runtime_mut().report_tracked_write(durability);
     }
 
-    /// This method triggers cancellation.
-    /// If you invoke it while a snapshot exists, it
-    /// will block until that snapshot is dropped -- if that snapshot
-    /// is owned by the current thread, this could trigger deadlock.
-    fn trigger_cancellation(&mut self) {
-        let _ = self.zalsa_mut();
-    }
-
     /// Reports that the query depends on some state unknown to salsa.
     ///
     /// Queries which report untracked reads will be re-executed in the next
@@ -115,30 +80,29 @@ pub trait Database: Send + ZalsaDatabase + AsDynDatabase {
         crate::attach::attach(self, || op(self))
     }
 
-    #[cold]
-    #[inline(never)]
     #[doc(hidden)]
-    fn zalsa_register_downcaster(&self) -> &DatabaseDownCaster<dyn Database> {
+    #[inline(always)]
-        self.zalsa().views().downcaster_for::<dyn Database>()
+    fn zalsa_register_downcaster(&self) {
         // The no-op downcaster is special cased in view caster construction.
     }
 
     #[doc(hidden)]
     #[inline(always)]
-    fn downcast(&self) -> &dyn Database
+    unsafe fn downcast(db: &dyn Database) -> &dyn Database
     where
         Self: Sized,
     {
         // No-op
-        self
+        db
     }
 }
 
 /// Upcast to a `dyn Database`.
 ///
-/// Only required because upcasting does not work for unsized generic parameters.
+/// Only required because upcasts not yet stabilized (*grr*).
 pub trait AsDynDatabase {
     fn as_dyn_database(&self) -> &dyn Database;
+    fn as_dyn_database_mut(&mut self) -> &mut dyn Database;
 }
 
 impl<T: Database> AsDynDatabase for T {
@@ -146,12 +110,30 @@ impl<T: Database> AsDynDatabase for T {
     fn as_dyn_database(&self) -> &dyn Database {
         self
     }
+
+    #[inline(always)]
+    fn as_dyn_database_mut(&mut self) -> &mut dyn Database {
+        self
+    }
 }
 
 pub fn current_revision<Db: ?Sized + Database>(db: &Db) -> Revision {
     db.zalsa().current_revision()
 }
 
+impl dyn Database {
+    /// Upcasts `self` to the given view.
+    ///
+    /// # Panics
+    ///
+    /// If the view has not been added to the database (see [`crate::views::Views`]).
+    #[track_caller]
+    pub fn as_view<DbView: ?Sized + Database>(&self) -> &DbView {
+        let views = self.zalsa().views();
+        views.downcaster_for().downcast(self)
+    }
+}
+
 #[cfg(feature = "salsa_unstable")]
 pub use memory_usage::IngredientInfo;
 
@@ -172,24 +154,17 @@ mod memory_usage {
                     let mut size_of_fields = 0;
                     let mut size_of_metadata = 0;
                     let mut instances = 0;
-                    let mut heap_size_of_fields = None;
 
                     for slot in ingredient.memory_usage(self)? {
                         instances += 1;
                         size_of_fields += slot.size_of_fields;
                         size_of_metadata += slot.size_of_metadata;
-
-                        if let Some(slot_heap_size) = slot.heap_size_of_fields {
-                            heap_size_of_fields =
-                                Some(heap_size_of_fields.unwrap_or_default() + slot_heap_size);
-                        }
                     }
 
                     Some(IngredientInfo {
                         count: instances,
                         size_of_fields,
                         size_of_metadata,
-                        heap_size_of_fields,
                         debug_name: ingredient.debug_name(),
                     })
                 })
@@ -218,11 +193,6 @@ mod memory_usage {
                         info.count += 1;
                         info.size_of_fields += memo.output.size_of_fields;
                         info.size_of_metadata += memo.output.size_of_metadata;
-
-                        if let Some(memo_heap_size) = memo.output.heap_size_of_fields {
-                            info.heap_size_of_fields =
-                                Some(info.heap_size_of_fields.unwrap_or_default() + memo_heap_size);
-                        }
                     }
                 }
             }
@@ -238,7 +208,6 @@ mod memory_usage {
         count: usize,
         size_of_metadata: usize,
         size_of_fields: usize,
-        heap_size_of_fields: Option<usize>,
     }
 
     impl IngredientInfo {
@@ -247,18 +216,11 @@ mod memory_usage {
             self.debug_name
         }
 
-        /// Returns the total stack size of the fields of any instances of this ingredient, in bytes.
+        /// Returns the total size of the fields of any instances of this ingredient, in bytes.
         pub fn size_of_fields(&self) -> usize {
             self.size_of_fields
         }
 
-        /// Returns the total heap size of the fields of any instances of this ingredient, in bytes.
-        ///
-        /// Returns `None` if the ingredient doesn't specify a `heap_size` function.
-        pub fn heap_size_of_fields(&self) -> Option<usize> {
-            self.heap_size_of_fields
-        }
-
         /// Returns the total size of Salsa metadata of any instances of this ingredient, in bytes.
         pub fn size_of_metadata(&self) -> usize {
             self.size_of_metadata
@@ -275,7 +237,6 @@ mod memory_usage {
         pub(crate) debug_name: &'static str,
         pub(crate) size_of_metadata: usize,
         pub(crate) size_of_fields: usize,
-        pub(crate) heap_size_of_fields: Option<usize>,
         pub(crate) memos: Vec<MemoInfo>,
     }
 

src/database_impl.rs

@@ -16,7 +16,7 @@ impl Default for DatabaseImpl {
             // Default behavior: tracing debug log the event.
             storage: Storage::new(if tracing::enabled!(Level::DEBUG) {
                 Some(Box::new(|event| {
-                    crate::tracing::debug!("salsa_event({:?})", event)
+                    tracing::debug!("salsa_event({:?})", event)
                 }))
             } else {
                 None

src/function.rs

@@ -3,14 +3,12 @@ use std::any::Any;
 use std::fmt;
 use std::ptr::NonNull;
 use std::sync::atomic::Ordering;
-use std::sync::OnceLock;
 pub(crate) use sync::SyncGuard;
 
+use crate::accumulator::accumulated_map::{AccumulatedMap, InputAccumulatedValues};
 use crate::cycle::{
-    empty_cycle_heads, CycleHeadKeys, CycleHeads, CycleRecoveryAction, CycleRecoveryStrategy,
+    empty_cycle_heads, CycleHeads, CycleRecoveryAction, CycleRecoveryStrategy, ProvisionalStatus,
-    ProvisionalStatus,
 };
-use crate::database::RawDatabase;
 use crate::function::delete::DeletedEntries;
 use crate::function::sync::{ClaimResult, SyncTable};
 use crate::ingredient::{Ingredient, WaitForResult};
@@ -23,9 +21,8 @@ use crate::table::Table;
 use crate::views::DatabaseDownCaster;
 use crate::zalsa::{IngredientIndex, MemoIngredientIndex, Zalsa};
 use crate::zalsa_local::QueryOriginRef;
-use crate::{Id, Revision};
+use crate::{Database, Id, Revision};
 
-#[cfg(feature = "accumulator")]
 mod accumulated;
 mod backdate;
 mod delete;
@@ -70,13 +67,14 @@ pub trait Configuration: Any {
     /// This invokes user code in form of the `Eq` impl.
     fn values_equal<'db>(old_value: &Self::Output<'db>, new_value: &Self::Output<'db>) -> bool;
 
+    // FIXME: This should take a `&Zalsa`
     /// Convert from the id used internally to the value that execute is expecting.
     /// This is a no-op if the input to the function is a salsa struct.
-    fn id_to_input(zalsa: &Zalsa, key: Id) -> Self::Input<'_>;
+    fn id_to_input(db: &Self::DbView, key: Id) -> Self::Input<'_>;
 
     /// Returns the size of any heap allocations in the output value, in bytes.
-    fn heap_size(_value: &Self::Output<'_>) -> Option<usize> {
+    fn heap_size(_value: &Self::Output<'_>) -> usize {
-        None
+        0
     }
 
     /// Invoked when we need to compute the value for the given key, either because we've never
@@ -125,13 +123,13 @@ pub struct IngredientImpl<C: Configuration> {
     /// Used to find memos to throw out when we have too many memoized values.
     lru: lru::Lru,
 
-    /// An downcaster to `C::DbView`.
+    /// A downcaster from `dyn Database` to `C::DbView`.
     ///
     /// # Safety
     ///
     /// The supplied database must be be the same as the database used to construct the [`Views`]
     /// instances that this downcaster was derived from.
-    view_caster: OnceLock<DatabaseDownCaster<C::DbView>>,
+    view_caster: DatabaseDownCaster<C::DbView>,
 
     sync_table: SyncTable,
 
@@ -158,30 +156,18 @@ where
         index: IngredientIndex,
         memo_ingredient_indices: <C::SalsaStruct<'static> as SalsaStructInDb>::MemoIngredientMap,
         lru: usize,
+        view_caster: DatabaseDownCaster<C::DbView>,
     ) -> Self {
         Self {
             index,
             memo_ingredient_indices,
             lru: lru::Lru::new(lru),
             deleted_entries: Default::default(),
-            view_caster: OnceLock::new(),
+            view_caster,
             sync_table: SyncTable::new(index),
         }
     }
 
-    /// Set the view-caster for this tracked function ingredient, if it has
-    /// not already been initialized.
-    #[inline]
-    pub fn get_or_init(
-        &self,
-        view_caster: impl FnOnce() -> DatabaseDownCaster<C::DbView>,
-    ) -> &Self {
-        // Note that we must set this lazily as we don't have access to the database
-        // type when ingredients are registered into the `Zalsa`.
-        self.view_caster.get_or_init(view_caster);
-        self
-    }
-
     #[inline]
     pub fn database_key_index(&self, key: Id) -> DatabaseKeyIndex {
         DatabaseKeyIndex::new(self.index, key)
@@ -240,12 +226,6 @@ where
     fn memo_ingredient_index(&self, zalsa: &Zalsa, id: Id) -> MemoIngredientIndex {
         self.memo_ingredient_indices.get_zalsa_id(zalsa, id)
     }
-
-    fn view_caster(&self) -> &DatabaseDownCaster<C::DbView> {
-        self.view_caster
-            .get()
-            .expect("tracked function ingredients cannot be accessed before calling `init`")
-    }
 }
 
 impl<C> Ingredient for IngredientImpl<C>
@@ -262,14 +242,13 @@ where
 
     unsafe fn maybe_changed_after(
         &self,
-        _zalsa: &crate::zalsa::Zalsa,
+        db: &dyn Database,
-        db: RawDatabase<'_>,
         input: Id,
         revision: Revision,
-        cycle_heads: &mut CycleHeadKeys,
+        cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
         // SAFETY: The `db` belongs to the ingredient as per caller invariant
-        let db = unsafe { self.view_caster().downcast_unchecked(db) };
+        let db = unsafe { self.view_caster.downcast_unchecked(db) };
         self.maybe_changed_after(db, input, revision, cycle_heads)
     }
 
@@ -311,7 +290,7 @@ where
     fn wait_for<'me>(&'me self, zalsa: &'me Zalsa, key_index: Id) -> WaitForResult<'me> {
         match self.sync_table.try_claim(zalsa, key_index) {
             ClaimResult::Running(blocked_on) => WaitForResult::Running(blocked_on),
-            ClaimResult::Cycle => WaitForResult::Cycle,
+            ClaimResult::Cycle { same_thread } => WaitForResult::Cycle { same_thread },
             ClaimResult::Claimed(_) => WaitForResult::Available,
         }
     }
@@ -360,11 +339,7 @@ where
         C::DEBUG_NAME
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        unreachable!("function does not allocate pages")
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
         unreachable!("function does not allocate pages")
     }
 
@@ -372,17 +347,12 @@ where
         C::CYCLE_STRATEGY
     }
 
-    #[cfg(feature = "accumulator")]
+    fn accumulated<'db>(
-    unsafe fn accumulated<'db>(
         &'db self,
-        db: RawDatabase<'db>,
+        db: &'db dyn Database,
         key_index: Id,
-    ) -> (
+    ) -> (Option<&'db AccumulatedMap>, InputAccumulatedValues) {
-        Option<&'db crate::accumulator::accumulated_map::AccumulatedMap>,
+        let db = self.view_caster.downcast(db);
-        crate::accumulator::accumulated_map::InputAccumulatedValues,
-    ) {
-        // SAFETY: The `db` belongs to the ingredient as per caller invariant
-        let db = unsafe { self.view_caster().downcast_unchecked(db) };
         self.accumulated_map(db, key_index)
     }
 }

src/function/accumulated.rs

@@ -4,7 +4,7 @@ use crate::function::{Configuration, IngredientImpl};
 use crate::hash::FxHashSet;
 use crate::zalsa::ZalsaDatabase;
 use crate::zalsa_local::QueryOriginRef;
-use crate::{DatabaseKeyIndex, Id};
+use crate::{AsDynDatabase, DatabaseKeyIndex, Id};
 
 impl<C> IngredientImpl<C>
 where
@@ -37,8 +37,9 @@ where
         let mut output = vec![];
 
         // First ensure the result is up to date
-        self.fetch(db, zalsa, zalsa_local, key);
+        self.fetch(db, key);
 
+        let db = db.as_dyn_database();
         let db_key = self.database_key_index(key);
         let mut visited: FxHashSet<DatabaseKeyIndex> = FxHashSet::default();
         let mut stack: Vec<DatabaseKeyIndex> = vec![db_key];
@@ -53,9 +54,7 @@ where
 
             let ingredient = zalsa.lookup_ingredient(k.ingredient_index());
             // Extend `output` with any values accumulated by `k`.
-            // SAFETY: `db` owns the `ingredient`
+            let (accumulated_map, input) = ingredient.accumulated(db, k.key_index());
-            let (accumulated_map, input) =
-                unsafe { ingredient.accumulated(db.into(), k.key_index()) };
             if let Some(accumulated_map) = accumulated_map {
                 accumulated_map.extend_with_accumulated(accumulator.index(), &mut output);
             }

src/function/backdate.rs

@@ -34,7 +34,7 @@ where
             if revisions.durability >= old_memo.revisions.durability
                 && C::values_equal(old_value, value)
             {
-                crate::tracing::debug!(
+                tracing::debug!(
                     "{index:?} value is equal, back-dating to {:?}",
                     old_memo.revisions.changed_at,
                 );

src/function/execute.rs

@@ -4,7 +4,7 @@ use crate::function::{Configuration, IngredientImpl};
 use crate::sync::atomic::{AtomicBool, Ordering};
 use crate::zalsa::{MemoIngredientIndex, Zalsa, ZalsaDatabase};
 use crate::zalsa_local::{ActiveQueryGuard, QueryRevisions};
-use crate::{Event, EventKind, Id};
+use crate::{Event, EventKind, Id, Revision};
 
 impl<C> IngredientImpl<C>
 where
@@ -29,7 +29,7 @@ where
         let database_key_index = active_query.database_key_index;
         let id = database_key_index.key_index();
 
-        crate::tracing::info!("{:?}: executing query", database_key_index);
+        tracing::info!("{:?}: executing query", database_key_index);
         let zalsa = db.zalsa();
 
         zalsa.event(&|| {
@@ -41,11 +41,16 @@ where
 
         let (new_value, mut revisions) = match C::CYCLE_STRATEGY {
             CycleRecoveryStrategy::Panic => {
-                Self::execute_query(db, zalsa, active_query, opt_old_memo, id)
+                Self::execute_query(db, active_query, opt_old_memo, zalsa.current_revision(), id)
             }
             CycleRecoveryStrategy::FallbackImmediate => {
-                let (mut new_value, mut revisions) =
+                let (mut new_value, mut revisions) = Self::execute_query(
-                    Self::execute_query(db, zalsa, active_query, opt_old_memo, id);
+                    db,
+                    active_query,
+                    opt_old_memo,
+                    zalsa.current_revision(),
+                    id,
+                );
 
                 if let Some(cycle_heads) = revisions.cycle_heads_mut() {
                     // Did the new result we got depend on our own provisional value, in a cycle?
@@ -72,7 +77,7 @@ where
                     let active_query = db
                         .zalsa_local()
                         .push_query(database_key_index, IterationCount::initial());
-                    new_value = C::cycle_initial(db, C::id_to_input(zalsa, id));
+                    new_value = C::cycle_initial(db, C::id_to_input(db, id));
                     revisions = active_query.pop();
                     // We need to set `cycle_heads` and `verified_final` because it needs to propagate to the callers.
                     // When verifying this, we will see we have fallback and mark ourselves verified.
@@ -131,8 +136,13 @@ where
         let mut opt_last_provisional: Option<&Memo<'db, C>> = None;
         loop {
             let previous_memo = opt_last_provisional.or(opt_old_memo);
-            let (mut new_value, mut revisions) =
+            let (mut new_value, mut revisions) = Self::execute_query(
-                Self::execute_query(db, zalsa, active_query, previous_memo, id);
+                db,
+                active_query,
+                previous_memo,
+                zalsa.current_revision(),
+                id,
+            );
 
             // Did the new result we got depend on our own provisional value, in a cycle?
             if let Some(cycle_heads) = revisions
@@ -159,7 +169,7 @@ where
                 };
                 // SAFETY: The `LRU` does not run mid-execution, so the value remains filled
                 let last_provisional_value = unsafe { last_provisional_value.unwrap_unchecked() };
-                crate::tracing::debug!(
+                tracing::debug!(
                     "{database_key_index:?}: execute: \
                         I am a cycle head, comparing last provisional value with new value"
                 );
@@ -182,11 +192,11 @@ where
                         db,
                         &new_value,
                         iteration_count.as_u32(),
-                        C::id_to_input(zalsa, id),
+                        C::id_to_input(db, id),
                     ) {
                         crate::CycleRecoveryAction::Iterate => {}
                         crate::CycleRecoveryAction::Fallback(fallback_value) => {
-                            crate::tracing::debug!(
+                            tracing::debug!(
                                 "{database_key_index:?}: execute: user cycle_fn says to fall back"
                             );
                             new_value = fallback_value;
@@ -210,7 +220,7 @@ where
                     });
                     cycle_heads.update_iteration_count(database_key_index, iteration_count);
                     revisions.update_iteration_count(iteration_count);
-                    crate::tracing::debug!(
+                    tracing::debug!(
                         "{database_key_index:?}: execute: iterate again, revisions: {revisions:#?}"
                     );
                     opt_last_provisional = Some(self.insert_memo(
@@ -226,7 +236,7 @@ where
 
                     continue;
                 }
-                crate::tracing::debug!(
+                tracing::debug!(
                     "{database_key_index:?}: execute: fixpoint iteration has a final value"
                 );
                 cycle_heads.remove(&database_key_index);
@@ -237,9 +247,7 @@ where
                 }
             }
 
-            crate::tracing::debug!(
+            tracing::debug!("{database_key_index:?}: execute: result.revisions = {revisions:#?}");
-                "{database_key_index:?}: execute: result.revisions = {revisions:#?}"
-            );
 
             break (new_value, revisions);
         }
@@ -248,9 +256,9 @@ where
     #[inline]
     fn execute_query<'db>(
         db: &'db C::DbView,
-        zalsa: &'db Zalsa,
         active_query: ActiveQueryGuard<'db>,
         opt_old_memo: Option<&Memo<'db, C>>,
+        current_revision: Revision,
         id: Id,
     ) -> (C::Output<'db>, QueryRevisions) {
         if let Some(old_memo) = opt_old_memo {
@@ -265,16 +273,14 @@ where
             // * ensure that tracked struct created during the previous iteration
             //   (and are owned by the query) are alive even if the query in this iteration no longer creates them.
             // * ensure the final returned memo depends on all inputs from all iterations.
-            if old_memo.may_be_provisional()
+            if old_memo.may_be_provisional() && old_memo.verified_at.load() == current_revision {
-                && old_memo.verified_at.load() == zalsa.current_revision()
-            {
                 active_query.seed_iteration(&old_memo.revisions);
             }
         }
 
         // Query was not previously executed, or value is potentially
         // stale, or value is absent. Let's execute!
-        let new_value = C::execute(db, C::id_to_input(zalsa, id));
+        let new_value = C::execute(db, C::id_to_input(db, id));
 
         (new_value, active_query.pop())
     }

src/function/fetch.rs

@@ -1,31 +1,24 @@
-use crate::cycle::{CycleHeadKeys, CycleHeads, CycleRecoveryStrategy, IterationCount};
+use crate::cycle::{CycleHeads, CycleRecoveryStrategy, IterationCount};
 use crate::function::memo::Memo;
 use crate::function::sync::ClaimResult;
 use crate::function::{Configuration, IngredientImpl, VerifyResult};
-use crate::zalsa::{MemoIngredientIndex, Zalsa};
+use crate::zalsa::{MemoIngredientIndex, Zalsa, ZalsaDatabase};
 use crate::zalsa_local::{QueryRevisions, ZalsaLocal};
-use crate::{DatabaseKeyIndex, Id};
+use crate::Id;
 
 impl<C> IngredientImpl<C>
 where
     C: Configuration,
 {
-    pub fn fetch<'db>(
+    pub fn fetch<'db>(&'db self, db: &'db C::DbView, id: Id) -> &'db C::Output<'db> {
-        &'db self,
+        let (zalsa, zalsa_local) = db.zalsas();
-        db: &'db C::DbView,
-        zalsa: &'db Zalsa,
-        zalsa_local: &'db ZalsaLocal,
-        id: Id,
-    ) -> &'db C::Output<'db> {
         zalsa.unwind_if_revision_cancelled(zalsa_local);
 
         let database_key_index = self.database_key_index(id);
-
         #[cfg(debug_assertions)]
-        let _span = crate::tracing::debug_span!("fetch", query = ?database_key_index).entered();
+        let _span = tracing::debug_span!("fetch", query = ?database_key_index).entered();
 
         let memo = self.refresh_memo(db, zalsa, zalsa_local, id);
-
         // SAFETY: We just refreshed the memo so it is guaranteed to contain a value now.
         let memo_value = unsafe { memo.value.as_ref().unwrap_unchecked() };
 
@@ -35,11 +28,9 @@ where
             database_key_index,
             memo.revisions.durability,
             memo.revisions.changed_at,
-            memo.cycle_heads(),
-            #[cfg(feature = "accumulator")]
             memo.revisions.accumulated().is_some(),
-            #[cfg(feature = "accumulator")]
             &memo.revisions.accumulated_inputs,
+            memo.cycle_heads(),
         );
 
         memo_value
@@ -130,7 +121,6 @@ where
         let database_key_index = self.database_key_index(id);
         // Try to claim this query: if someone else has claimed it already, go back and start again.
         let claim_guard = match self.sync_table.try_claim(zalsa, id) {
-            ClaimResult::Claimed(guard) => guard,
             ClaimResult::Running(blocked_on) => {
                 blocked_on.block_on(zalsa);
 
@@ -147,15 +137,72 @@ where
                 return None;
             }
             ClaimResult::Cycle { .. } => {
-                return Some(self.fetch_cold_cycle(
+                // check if there's a provisional value for this query
-                    zalsa,
+                // Note we don't `validate_may_be_provisional` the memo here as we want to reuse an
-                    zalsa_local,
+                // existing provisional memo if it exists
-                    db,
+                let memo_guard = self.get_memo_from_table_for(zalsa, id, memo_ingredient_index);
-                    id,
+                if let Some(memo) = memo_guard {
-                    database_key_index,
+                    if memo.value.is_some()
-                    memo_ingredient_index,
+                        && memo.revisions.cycle_heads().contains(&database_key_index)
-                ));
+                    {
+                        let can_shallow_update =
+                            self.shallow_verify_memo(zalsa, database_key_index, memo);
+                        if can_shallow_update.yes() {
+                            self.update_shallow(
+                                zalsa,
+                                database_key_index,
+                                memo,
+                                can_shallow_update,
+                            );
+                            // SAFETY: memo is present in memo_map.
+                            return unsafe { Some(self.extend_memo_lifetime(memo)) };
+                        }
+                    }
+                }
+                // no provisional value; create/insert/return initial provisional value
+                return match C::CYCLE_STRATEGY {
+                    CycleRecoveryStrategy::Panic => zalsa_local.with_query_stack(|stack| {
+                        panic!(
+                            "dependency graph cycle when querying {database_key_index:#?}, \
+                            set cycle_fn/cycle_initial to fixpoint iterate.\n\
+                            Query stack:\n{stack:#?}",
+                        );
+                    }),
+                    CycleRecoveryStrategy::Fixpoint => {
+                        tracing::debug!(
+                            "hit cycle at {database_key_index:#?}, \
+                            inserting and returning fixpoint initial value"
+                        );
+                        let revisions = QueryRevisions::fixpoint_initial(database_key_index);
+                        let initial_value = C::cycle_initial(db, C::id_to_input(db, id));
+                        Some(self.insert_memo(
+                            zalsa,
+                            id,
+                            Memo::new(Some(initial_value), zalsa.current_revision(), revisions),
+                            memo_ingredient_index,
+                        ))
+                    }
+                    CycleRecoveryStrategy::FallbackImmediate => {
+                        tracing::debug!(
+                            "hit a `FallbackImmediate` cycle at {database_key_index:#?}"
+                        );
+                        let active_query =
+                            zalsa_local.push_query(database_key_index, IterationCount::initial());
+                        let fallback_value = C::cycle_initial(db, C::id_to_input(db, id));
+                        let mut revisions = active_query.pop();
+                        revisions.set_cycle_heads(CycleHeads::initial(database_key_index));
+                        // We need this for `cycle_heads()` to work. We will unset this in the outer `execute()`.
+                        *revisions.verified_final.get_mut() = false;
+                        Some(self.insert_memo(
+                            zalsa,
+                            id,
+                            Memo::new(Some(fallback_value), zalsa.current_revision(), revisions),
+                            memo_ingredient_index,
+                        ))
+                    }
+                };
             }
+            ClaimResult::Claimed(guard) => guard,
         };
 
         // Now that we've claimed the item, check again to see if there's a "hot" value.
@@ -163,8 +210,8 @@ where
 
         if let Some(old_memo) = opt_old_memo {
             if old_memo.value.is_some() {
-                let mut cycle_heads = CycleHeadKeys::new();
+                let mut cycle_heads = CycleHeads::default();
-                if let VerifyResult::Unchanged { .. } =
+                if let VerifyResult::Unchanged(_) =
                     self.deep_verify_memo(db, zalsa, old_memo, database_key_index, &mut cycle_heads)
                 {
                     if cycle_heads.is_empty() {
@@ -213,77 +260,4 @@ where
 
         Some(memo)
     }
-
-    #[cold]
-    #[inline(never)]
-    fn fetch_cold_cycle<'db>(
-        &'db self,
-        zalsa: &'db Zalsa,
-        zalsa_local: &'db ZalsaLocal,
-        db: &'db C::DbView,
-        id: Id,
-        database_key_index: DatabaseKeyIndex,
-        memo_ingredient_index: MemoIngredientIndex,
-    ) -> &'db Memo<'db, C> {
-        // check if there's a provisional value for this query
-        // Note we don't `validate_may_be_provisional` the memo here as we want to reuse an
-        // existing provisional memo if it exists
-        let memo_guard = self.get_memo_from_table_for(zalsa, id, memo_ingredient_index);
-        if let Some(memo) = memo_guard {
-            if memo.value.is_some() && memo.revisions.cycle_heads().contains(&database_key_index) {
-                let can_shallow_update = self.shallow_verify_memo(zalsa, database_key_index, memo);
-                if can_shallow_update.yes() {
-                    self.update_shallow(zalsa, database_key_index, memo, can_shallow_update);
-                    // SAFETY: memo is present in memo_map.
-                    return unsafe { self.extend_memo_lifetime(memo) };
-                }
-            }
-        }
-
-        // no provisional value; create/insert/return initial provisional value
-        match C::CYCLE_STRATEGY {
-            // SAFETY: We do not access the query stack reentrantly.
-            CycleRecoveryStrategy::Panic => unsafe {
-                zalsa_local.with_query_stack_unchecked(|stack| {
-                    panic!(
-                        "dependency graph cycle when querying {database_key_index:#?}, \
-                    set cycle_fn/cycle_initial to fixpoint iterate.\n\
-                    Query stack:\n{stack:#?}",
-                    );
-                })
-            },
-            CycleRecoveryStrategy::Fixpoint => {
-                crate::tracing::debug!(
-                    "hit cycle at {database_key_index:#?}, \
-                    inserting and returning fixpoint initial value"
-                );
-                let revisions = QueryRevisions::fixpoint_initial(database_key_index);
-                let initial_value = C::cycle_initial(db, C::id_to_input(zalsa, id));
-                self.insert_memo(
-                    zalsa,
-                    id,
-                    Memo::new(Some(initial_value), zalsa.current_revision(), revisions),
-                    memo_ingredient_index,
-                )
-            }
-            CycleRecoveryStrategy::FallbackImmediate => {
-                crate::tracing::debug!(
-                    "hit a `FallbackImmediate` cycle at {database_key_index:#?}"
-                );
-                let active_query =
-                    zalsa_local.push_query(database_key_index, IterationCount::initial());
-                let fallback_value = C::cycle_initial(db, C::id_to_input(zalsa, id));
-                let mut revisions = active_query.pop();
-                revisions.set_cycle_heads(CycleHeads::initial(database_key_index));
-                // We need this for `cycle_heads()` to work. We will unset this in the outer `execute()`.
-                *revisions.verified_final.get_mut() = false;
-                self.insert_memo(
-                    zalsa,
-                    id,
-                    Memo::new(Some(fallback_value), zalsa.current_revision(), revisions),
-                    memo_ingredient_index,
-                )
-            }
-        }
-    }
 }

src/function/maybe_changed_after.rs

@@ -1,6 +1,5 @@
-#[cfg(feature = "accumulator")]
 use crate::accumulator::accumulated_map::InputAccumulatedValues;
-use crate::cycle::{CycleHeadKeys, CycleRecoveryStrategy, IterationCount, ProvisionalStatus};
+use crate::cycle::{CycleHeads, CycleRecoveryStrategy, IterationCount, ProvisionalStatus};
 use crate::function::memo::Memo;
 use crate::function::sync::ClaimResult;
 use crate::function::{Configuration, IngredientImpl};
@@ -8,7 +7,7 @@ use crate::key::DatabaseKeyIndex;
 use crate::sync::atomic::Ordering;
 use crate::zalsa::{MemoIngredientIndex, Zalsa, ZalsaDatabase};
 use crate::zalsa_local::{QueryEdgeKind, QueryOriginRef, ZalsaLocal};
-use crate::{Id, Revision};
+use crate::{AsDynDatabase as _, Id, Revision};
 
 /// Result of memo validation.
 pub enum VerifyResult {
@@ -19,10 +18,7 @@ pub enum VerifyResult {
     ///
     /// The inner value tracks whether the memo or any of its dependencies have an
     /// accumulated value.
-    Unchanged {
+    Unchanged(InputAccumulatedValues),
-        #[cfg(feature = "accumulator")]
-        accumulated: InputAccumulatedValues,
-    },
 }
 
 impl VerifyResult {
@@ -35,10 +31,7 @@ impl VerifyResult {
     }
 
     pub(crate) fn unchanged() -> Self {
-        Self::Unchanged {
+        Self::Unchanged(InputAccumulatedValues::Empty)
-            #[cfg(feature = "accumulator")]
-            accumulated: InputAccumulatedValues::Empty,
-        }
     }
 }
 
@@ -51,7 +44,7 @@ where
         db: &'db C::DbView,
         id: Id,
         revision: Revision,
-        cycle_heads: &mut CycleHeadKeys,
+        cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
         let (zalsa, zalsa_local) = db.zalsas();
         let memo_ingredient_index = self.memo_ingredient_index(zalsa, id);
@@ -60,9 +53,7 @@ where
         loop {
             let database_key_index = self.database_key_index(id);
 
-            crate::tracing::debug!(
+            tracing::debug!("{database_key_index:?}: maybe_changed_after(revision = {revision:?})");
-                "{database_key_index:?}: maybe_changed_after(revision = {revision:?})"
-            );
 
             // Check if we have a verified version: this is the hot path.
             let memo_guard = self.get_memo_from_table_for(zalsa, id, memo_ingredient_index);
@@ -78,10 +69,7 @@ where
                 return if memo.revisions.changed_at > revision {
                     VerifyResult::Changed
                 } else {
-                    VerifyResult::Unchanged {
+                    VerifyResult::Unchanged(memo.revisions.accumulated_inputs.load())
-                        #[cfg(feature = "accumulator")]
-                        accumulated: memo.revisions.accumulated_inputs.load(),
-                    }
                 };
             }
 
@@ -108,23 +96,35 @@ where
         key_index: Id,
         revision: Revision,
         memo_ingredient_index: MemoIngredientIndex,
-        cycle_heads: &mut CycleHeadKeys,
+        cycle_heads: &mut CycleHeads,
     ) -> Option<VerifyResult> {
         let database_key_index = self.database_key_index(key_index);
 
         let _claim_guard = match self.sync_table.try_claim(zalsa, key_index) {
-            ClaimResult::Claimed(guard) => guard,
             ClaimResult::Running(blocked_on) => {
                 blocked_on.block_on(zalsa);
                 return None;
             }
-            ClaimResult::Cycle { .. } => {
+            ClaimResult::Cycle { .. } => match C::CYCLE_STRATEGY {
-                return Some(self.maybe_changed_after_cold_cycle(
+                CycleRecoveryStrategy::Panic => db.zalsa_local().with_query_stack(|stack| {
-                    db,
+                    panic!(
-                    database_key_index,
+                        "dependency graph cycle when validating {database_key_index:#?}, \
-                    cycle_heads,
+                        set cycle_fn/cycle_initial to fixpoint iterate.\n\
-                ))
+                        Query stack:\n{stack:#?}",
-            }
+                    );
+                }),
+                CycleRecoveryStrategy::FallbackImmediate => {
+                    return Some(VerifyResult::unchanged());
+                }
+                CycleRecoveryStrategy::Fixpoint => {
+                    tracing::debug!(
+                        "hit cycle at {database_key_index:?} in `maybe_changed_after`,  returning fixpoint initial value",
+                    );
+                    cycle_heads.push_initial(database_key_index);
+                    return Some(VerifyResult::unchanged());
+                }
+            },
+            ClaimResult::Claimed(guard) => guard,
         };
         // Load the current memo, if any.
         let Some(old_memo) = self.get_memo_from_table_for(zalsa, key_index, memo_ingredient_index)
@@ -132,7 +132,7 @@ where
             return Some(VerifyResult::Changed);
         };
 
-        crate::tracing::debug!(
+        tracing::debug!(
             "{database_key_index:?}: maybe_changed_after_cold, successful claim, \
                 revision = {revision:?}, old_memo = {old_memo:#?}",
             old_memo = old_memo.tracing_debug()
@@ -141,18 +141,11 @@ where
         // Check if the inputs are still valid. We can just compare `changed_at`.
         let deep_verify =
             self.deep_verify_memo(db, zalsa, old_memo, database_key_index, cycle_heads);
-        if let VerifyResult::Unchanged {
+        if let VerifyResult::Unchanged(accumulated_inputs) = deep_verify {
-            #[cfg(feature = "accumulator")]
-                accumulated: accumulated_inputs,
-        } = deep_verify
-        {
             return Some(if old_memo.revisions.changed_at > revision {
                 VerifyResult::Changed
             } else {
-                VerifyResult::Unchanged {
+                VerifyResult::Unchanged(accumulated_inputs)
-                    #[cfg(feature = "accumulator")]
-                    accumulated: accumulated_inputs,
-                }
             });
         }
 
@@ -176,13 +169,10 @@ where
             return Some(if changed_at > revision {
                 VerifyResult::Changed
             } else {
-                VerifyResult::Unchanged {
+                VerifyResult::Unchanged(match memo.revisions.accumulated() {
-                    #[cfg(feature = "accumulator")]
+                    Some(_) => InputAccumulatedValues::Any,
-                    accumulated: match memo.revisions.accumulated() {
+                    None => memo.revisions.accumulated_inputs.load(),
-                        Some(_) => InputAccumulatedValues::Any,
+                })
-                        None => memo.revisions.accumulated_inputs.load(),
-                    },
-                }
             });
         }
 
@@ -190,36 +180,6 @@ where
         Some(VerifyResult::Changed)
     }
 
-    #[cold]
-    #[inline(never)]
-    fn maybe_changed_after_cold_cycle<'db>(
-        &'db self,
-        db: &'db C::DbView,
-        database_key_index: DatabaseKeyIndex,
-        cycle_heads: &mut CycleHeadKeys,
-    ) -> VerifyResult {
-        match C::CYCLE_STRATEGY {
-            // SAFETY: We do not access the query stack reentrantly.
-            CycleRecoveryStrategy::Panic => unsafe {
-                db.zalsa_local().with_query_stack_unchecked(|stack| {
-                    panic!(
-                        "dependency graph cycle when validating {database_key_index:#?}, \
-                    set cycle_fn/cycle_initial to fixpoint iterate.\n\
-                    Query stack:\n{stack:#?}",
-                    );
-                })
-            },
-            CycleRecoveryStrategy::FallbackImmediate => VerifyResult::unchanged(),
-            CycleRecoveryStrategy::Fixpoint => {
-                crate::tracing::debug!(
-                    "hit cycle at {database_key_index:?} in `maybe_changed_after`,  returning fixpoint initial value",
-                );
-                cycle_heads.insert(database_key_index);
-                VerifyResult::unchanged()
-            }
-        }
-    }
-
     /// `Some` if the memo's value and `changed_at` time is still valid in this revision.
     /// Does only a shallow O(1) check, doesn't walk the dependencies.
     ///
@@ -234,7 +194,7 @@ where
         database_key_index: DatabaseKeyIndex,
         memo: &Memo<'_, C>,
     ) -> ShallowUpdate {
-        crate::tracing::debug!(
+        tracing::debug!(
             "{database_key_index:?}: shallow_verify_memo(memo = {memo:#?})",
             memo = memo.tracing_debug()
         );
@@ -247,7 +207,7 @@ where
         }
 
         let last_changed = zalsa.last_changed_revision(memo.revisions.durability);
-        crate::tracing::debug!(
+        tracing::debug!(
             "{database_key_index:?}: check_durability(memo = {memo:#?}, last_changed={:?} <= verified_at={:?}) = {:?}",
             last_changed,
             verified_at,
@@ -282,7 +242,7 @@ where
     ///   cycle heads have all been finalized.
     /// * provisional memos that have been created in the same revision and iteration and are part of the same cycle.
     #[inline]
-    fn validate_may_be_provisional(
+    pub(super) fn validate_may_be_provisional(
         &self,
         zalsa: &Zalsa,
         zalsa_local: &ZalsaLocal,
@@ -303,7 +263,7 @@ where
         database_key_index: DatabaseKeyIndex,
         memo: &Memo<'_, C>,
     ) -> bool {
-        crate::tracing::trace!(
+        tracing::trace!(
             "{database_key_index:?}: validate_provisional(memo = {memo:#?})",
             memo = memo.tracing_debug()
         );
@@ -357,14 +317,14 @@ where
     /// If this is a provisional memo, validate that it was cached in the same iteration of the
     /// same cycle(s) that we are still executing. If so, it is valid for reuse. This avoids
     /// runaway re-execution of the same queries within a fixpoint iteration.
-    fn validate_same_iteration(
+    pub(super) fn validate_same_iteration(
         &self,
         zalsa: &Zalsa,
         zalsa_local: &ZalsaLocal,
         database_key_index: DatabaseKeyIndex,
         memo: &Memo<'_, C>,
     ) -> bool {
-        crate::tracing::trace!(
+        tracing::trace!(
             "{database_key_index:?}: validate_same_iteration(memo = {memo:#?})",
             memo = memo.tracing_debug()
         );
@@ -374,65 +334,32 @@ where
             return true;
         }
 
-        // SAFETY: We do not access the query stack reentrantly.
+        zalsa_local.with_query_stack(|stack| {
-        unsafe {
+            cycle_heads.iter().all(|cycle_head| {
-            zalsa_local.with_query_stack_unchecked(|stack| {
+                stack
-                cycle_heads.iter().all(|cycle_head| {
+                    .iter()
-                    stack
+                    .rev()
-                        .iter()
+                    .find(|query| query.database_key_index == cycle_head.database_key_index)
-                        .rev()
+                    .map(|query| query.iteration_count())
-                        .find(|query| query.database_key_index == cycle_head.database_key_index)
+                    .or_else(|| {
-                        .map(|query| query.iteration_count())
+                        // If this is a cycle head is owned by another thread that is blocked by this ingredient,
-                        .or_else(|| {
+                        // check if it has the same iteration count.
-                            // If the cycle head isn't on our stack because:
+                        let ingredient = zalsa
-                            //
+                            .lookup_ingredient(cycle_head.database_key_index.ingredient_index());
-                            // * another thread holds the lock on the cycle head (but it waits for the current query to complete)
+                        let wait_result =
-                            // * we're in `maybe_changed_after` because `maybe_changed_after` doesn't modify the cycle stack
+                            ingredient.wait_for(zalsa, cycle_head.database_key_index.key_index());
-                            //
-                            // check if the latest memo has the same iteration count.
 
-                            // However, we've to be careful to skip over fixpoint initial values:
+                        if !wait_result.is_cycle_with_other_thread() {
-                            // If the head is the memo we're trying to validate, always return `None`
+                            return None;
-                            // to force a re-execution of the query. This is necessary because the query
+                        }
-                            // has obviously not completed its iteration yet.
-                            //
-                            // This should be rare but the `cycle_panic` test fails on some platforms (mainly GitHub actions)
-                            // without this check. What happens there is that:
-                            //
-                            // * query a blocks on query b
-                            // * query b tries to claim a, fails to do so and inserts the fixpoint initial value
-                            // * query b completes and has `a` as head. It returns its query result Salsa blocks query b from
-                            //   exiting inside `block_on` (or the thread would complete before the cycle iteration is complete)
-                            // * query a resumes but panics because of the fixpoint iteration function
-                            // * query b resumes. It rexecutes its own query which then tries to fetch a (which depends on itself because it's a fixpoint initial value).
-                            //   Without this check, `validate_same_iteration` would return `true` because the latest memo for `a` is the fixpoint initial value.
-                            //   But it should return `false` so that query b's thread re-executes `a` (which then also causes the panic).
-                            //
-                            // That's why we always return `None` if the cycle head is the same as the current database key index.
-                            if cycle_head.database_key_index == database_key_index {
-                                return None;
-                            }
 
-                            let ingredient = zalsa.lookup_ingredient(
+                        let provisional_status = ingredient
-                                cycle_head.database_key_index.ingredient_index(),
+                            .provisional_status(zalsa, cycle_head.database_key_index.key_index())?;
-                            );
+                        provisional_status.iteration()
-                            let wait_result = ingredient
+                    })
-                                .wait_for(zalsa, cycle_head.database_key_index.key_index());
+                    == Some(cycle_head.iteration_count)
-
-                            if !wait_result.is_cycle() {
-                                return None;
-                            }
-
-                            let provisional_status = ingredient.provisional_status(
-                                zalsa,
-                                cycle_head.database_key_index.key_index(),
-                            )?;
-                            provisional_status.iteration()
-                        })
-                        == Some(cycle_head.iteration_count)
-                })
             })
-        }
+        })
     }
 
     /// VerifyResult::Unchanged if the memo's value and `changed_at` time is up-to-date in the
@@ -448,9 +375,9 @@ where
         zalsa: &Zalsa,
         old_memo: &Memo<'_, C>,
         database_key_index: DatabaseKeyIndex,
-        cycle_heads: &mut CycleHeadKeys,
+        cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
-        crate::tracing::debug!(
+        tracing::debug!(
             "{database_key_index:?}: deep_verify_memo(old_memo = {old_memo:#?})",
             old_memo = old_memo.tracing_debug()
         );
@@ -470,6 +397,32 @@ where
         }
 
         match old_memo.revisions.origin.as_ref() {
+            QueryOriginRef::Assigned(_) => {
+                // If the value was assigned by another query,
+                // and that query were up-to-date,
+                // then we would have updated the `verified_at` field already.
+                // So the fact that we are here means that it was not specified
+                // during this revision or is otherwise stale.
+                //
+                // Example of how this can happen:
+                //
+                // Conditionally specified queries
+                // where the value is specified
+                // in rev 1 but not in rev 2.
+                VerifyResult::Changed
+            }
+            // Return `Unchanged` similar to the initial value that we insert
+            // when we hit the cycle. Any dependencies accessed when creating the fixpoint initial
+            // are tracked by the outer query. Nothing should have changed assuming that the
+            // fixpoint initial function is deterministic.
+            QueryOriginRef::FixpointInitial => {
+                cycle_heads.push_initial(database_key_index);
+                VerifyResult::unchanged()
+            }
+            QueryOriginRef::DerivedUntracked(_) => {
+                // Untracked inputs? Have to assume that it changed.
+                VerifyResult::Changed
+            }
             QueryOriginRef::Derived(edges) => {
                 let is_provisional = old_memo.may_be_provisional();
 
@@ -479,7 +432,8 @@ where
                     return VerifyResult::Changed;
                 }
 
-                #[cfg(feature = "accumulator")]
+                let dyn_db = db.as_dyn_database();
+
                 let mut inputs = InputAccumulatedValues::Empty;
                 // Fully tracked inputs? Iterate over the inputs and check them, one by one.
                 //
@@ -491,18 +445,15 @@ where
                     match edge.kind() {
                         QueryEdgeKind::Input(dependency_index) => {
                             match dependency_index.maybe_changed_after(
-                                db.into(),
+                                dyn_db,
                                 zalsa,
                                 old_memo.verified_at.load(),
                                 cycle_heads,
                             ) {
                                 VerifyResult::Changed => return VerifyResult::Changed,
-                                #[cfg(feature = "accumulator")]
+                                VerifyResult::Unchanged(input_accumulated) => {
-                                VerifyResult::Unchanged { accumulated } => {
+                                    inputs |= input_accumulated;
-                                    inputs |= accumulated;
                                 }
-                                #[cfg(not(feature = "accumulator"))]
-                                VerifyResult::Unchanged { .. } => {}
                             }
                         }
                         QueryEdgeKind::Output(dependency_index) => {
@@ -557,7 +508,6 @@ where
                 // 1 and 3
                 if cycle_heads.is_empty() {
                     old_memo.mark_as_verified(zalsa, database_key_index);
-                    #[cfg(feature = "accumulator")]
                     old_memo.revisions.accumulated_inputs.store(inputs);
 
                     if is_provisional {
@@ -568,37 +518,7 @@ where
                     }
                 }
 
-                VerifyResult::Unchanged {
+                VerifyResult::Unchanged(inputs)
-                    #[cfg(feature = "accumulator")]
-                    accumulated: inputs,
-                }
-            }
-
-            QueryOriginRef::Assigned(_) => {
-                // If the value was assigned by another query,
-                // and that query were up-to-date,
-                // then we would have updated the `verified_at` field already.
-                // So the fact that we are here means that it was not specified
-                // during this revision or is otherwise stale.
-                //
-                // Example of how this can happen:
-                //
-                // Conditionally specified queries
-                // where the value is specified
-                // in rev 1 but not in rev 2.
-                VerifyResult::Changed
-            }
-            // Return `Unchanged` similar to the initial value that we insert
-            // when we hit the cycle. Any dependencies accessed when creating the fixpoint initial
-            // are tracked by the outer query. Nothing should have changed assuming that the
-            // fixpoint initial function is deterministic.
-            QueryOriginRef::FixpointInitial => {
-                cycle_heads.insert(database_key_index);
-                VerifyResult::unchanged()
-            }
-            QueryOriginRef::DerivedUntracked(_) => {
-                // Untracked inputs? Have to assume that it changed.
-                VerifyResult::Changed
             }
         }
     }

src/function/memo.rs

@@ -30,7 +30,7 @@ impl<C: Configuration> IngredientImpl<C> {
         let static_memo =
             unsafe { transmute::<NonNull<Memo<'db, C>>, NonNull<Memo<'static, C>>>(memo) };
         let old_static_memo = zalsa
-            .memo_table_for::<C::SalsaStruct<'_>>(id)
+            .memo_table_for(id)
             .insert(memo_ingredient_index, static_memo)?;
         // SAFETY: The table stores 'static memos (to support `Any`), the memos are in fact valid
         // for `'db` though as we delay their dropping to the end of a revision.
@@ -48,9 +48,7 @@ impl<C: Configuration> IngredientImpl<C> {
         id: Id,
         memo_ingredient_index: MemoIngredientIndex,
     ) -> Option<&'db Memo<'db, C>> {
-        let static_memo = zalsa
+        let static_memo = zalsa.memo_table_for(id).get(memo_ingredient_index)?;
-            .memo_table_for::<C::SalsaStruct<'_>>(id)
-            .get(memo_ingredient_index)?;
         // SAFETY: The table stores 'static memos (to support `Any`), the memos are in fact valid
         // for `'db` though as we delay their dropping to the end of a revision.
         Some(unsafe { transmute::<&Memo<'static, C>, &'db Memo<'db, C>>(static_memo.as_ref()) })
@@ -153,7 +151,7 @@ impl<'db, C: Configuration> Memo<'db, C> {
         } else {
             // all our cycle heads are complete; re-fetch
             // and we should get a non-provisional memo.
-            crate::tracing::debug!(
+            tracing::debug!(
                 "Retrying provisional memo {database_key_index:?} after awaiting cycle heads."
             );
             true
@@ -180,7 +178,7 @@ impl<'db, C: Configuration> Memo<'db, C> {
 
         #[inline(never)]
         fn block_on_heads_cold(zalsa: &Zalsa, heads: &CycleHeads) -> bool {
-            let _entered = crate::tracing::debug_span!("block_on_heads").entered();
+            let _entered = tracing::debug_span!("block_on_heads").entered();
             let mut cycle_heads = TryClaimCycleHeadsIter::new(zalsa, heads);
             let mut all_cycles = true;
 
@@ -209,7 +207,7 @@ impl<'db, C: Configuration> Memo<'db, C> {
     /// Unlike `block_on_heads`, this code does not block on any cycle head. Instead it returns `false` if
     /// claiming all cycle heads failed because one of them is running on another thread.
     pub(super) fn try_claim_heads(&self, zalsa: &Zalsa, zalsa_local: &ZalsaLocal) -> bool {
-        let _entered = crate::tracing::debug_span!("try_claim_heads").entered();
+        let _entered = tracing::debug_span!("try_claim_heads").entered();
         if self.all_cycles_on_stack(zalsa_local) {
             return true;
         }
@@ -236,17 +234,14 @@ impl<'db, C: Configuration> Memo<'db, C> {
             return true;
         }
 
-        // SAFETY: We do not access the query stack reentrantly.
+        zalsa_local.with_query_stack(|stack| {
-        unsafe {
+            cycle_heads.iter().all(|cycle_head| {
-            zalsa_local.with_query_stack_unchecked(|stack| {
+                stack
-                cycle_heads.iter().all(|cycle_head| {
+                    .iter()
-                    stack
+                    .rev()
-                        .iter()
+                    .any(|query| query.database_key_index == cycle_head.database_key_index)
-                        .rev()
-                        .any(|query| query.database_key_index == cycle_head.database_key_index)
-                })
             })
-        }
+        })
     }
 
     /// Cycle heads that should be propagated to dependent queries.
@@ -321,19 +316,14 @@ where
     #[cfg(feature = "salsa_unstable")]
     fn memory_usage(&self) -> crate::database::MemoInfo {
         let size_of = std::mem::size_of::<Memo<C>>() + self.revisions.allocation_size();
-        let heap_size = if let Some(value) = self.value.as_ref() {
+        let heap_size = self.value.as_ref().map(C::heap_size).unwrap_or(0);
-            C::heap_size(value)
-        } else {
-            Some(0)
-        };
 
         crate::database::MemoInfo {
             debug_name: C::DEBUG_NAME,
             output: crate::database::SlotInfo {
                 size_of_metadata: size_of - std::mem::size_of::<C::Output<'static>>(),
                 debug_name: std::any::type_name::<C::Output<'static>>(),
-                size_of_fields: std::mem::size_of::<C::Output<'static>>(),
+                size_of_fields: std::mem::size_of::<C::Output<'static>>() + heap_size,
-                heap_size_of_fields: heap_size,
                 memos: Vec::new(),
             },
         }
@@ -427,7 +417,7 @@ impl<'me> Iterator for TryClaimCycleHeadsIter<'me> {
             ProvisionalStatus::Final { .. } | ProvisionalStatus::FallbackImmediate => {
                 // This cycle is already finalized, so we don't need to wait on it;
                 // keep looping through cycle heads.
-                crate::tracing::trace!("Dependent cycle head {head:?} has been finalized.");
+                tracing::trace!("Dependent cycle head {head:?} has been finalized.");
                 Some(TryClaimHeadsResult::Finalized)
             }
             ProvisionalStatus::Provisional { .. } => {
@@ -435,11 +425,11 @@ impl<'me> Iterator for TryClaimCycleHeadsIter<'me> {
                     WaitForResult::Cycle { .. } => {
                         // We hit a cycle blocking on the cycle head; this means this query actively
                         // participates in the cycle and some other query is blocked on this thread.
-                        crate::tracing::debug!("Waiting for {head:?} results in a cycle");
+                        tracing::debug!("Waiting for {head:?} results in a cycle");
                         Some(TryClaimHeadsResult::Cycle)
                     }
                     WaitForResult::Running(running) => {
-                        crate::tracing::debug!("Ingredient {head:?} is running: {running:?}");
+                        tracing::debug!("Ingredient {head:?} is running: {running:?}");
 
                         Some(TryClaimHeadsResult::Running(RunningCycleHead {
                             inner: running,
@@ -461,9 +451,8 @@ mod _memory_usage {
     use crate::cycle::CycleRecoveryStrategy;
     use crate::ingredient::Location;
     use crate::plumbing::{IngredientIndices, MemoIngredientSingletonIndex, SalsaStructInDb};
-    use crate::table::memo::MemoTableWithTypes;
     use crate::zalsa::Zalsa;
-    use crate::{CycleRecoveryAction, Database, Id, Revision};
+    use crate::{CycleRecoveryAction, Database, Id};
 
     use std::any::TypeId;
     use std::num::NonZeroUsize;
@@ -477,17 +466,13 @@ mod _memory_usage {
     impl SalsaStructInDb for DummyStruct {
         type MemoIngredientMap = MemoIngredientSingletonIndex;
 
-        fn lookup_ingredient_index(_: &Zalsa) -> IngredientIndices {
+        fn lookup_or_create_ingredient_index(_: &Zalsa) -> IngredientIndices {
             unimplemented!()
         }
 
         fn cast(_: Id, _: TypeId) -> Option<Self> {
             unimplemented!()
         }
-
-        unsafe fn memo_table(_: &Zalsa, _: Id, _: Revision) -> MemoTableWithTypes<'_> {
-            unimplemented!()
-        }
     }
 
     struct DummyConfiguration;
@@ -505,7 +490,7 @@ mod _memory_usage {
             unimplemented!()
         }
 
-        fn id_to_input(_: &Zalsa, _: Id) -> Self::Input<'_> {
+        fn id_to_input(_: &Self::DbView, _: Id) -> Self::Input<'_> {
             unimplemented!()
         }
 

src/function/specify.rs

@@ -1,4 +1,3 @@
-#[cfg(feature = "accumulator")]
 use crate::accumulator::accumulated_map::InputAccumulatedValues;
 use crate::function::memo::Memo;
 use crate::function::{Configuration, IngredientImpl};
@@ -67,7 +66,6 @@ where
             changed_at: current_deps.changed_at,
             durability: current_deps.durability,
             origin: QueryOrigin::assigned(active_query_key),
-            #[cfg(feature = "accumulator")]
             accumulated_inputs: Default::default(),
             verified_final: AtomicBool::new(true),
             extra: QueryRevisionsExtra::default(),
@@ -85,7 +83,7 @@ where
             revisions,
         };
 
-        crate::tracing::debug!(
+        tracing::debug!(
             "specify: about to add memo {:#?} for key {:?}",
             memo.tracing_debug(),
             key
@@ -126,7 +124,6 @@ where
 
         let database_key_index = self.database_key_index(key);
         memo.mark_as_verified(zalsa, database_key_index);
-        #[cfg(feature = "accumulator")]
         memo.revisions
             .accumulated_inputs
             .store(InputAccumulatedValues::Empty);

src/function/sync.rs

@@ -20,7 +20,7 @@ pub(crate) enum ClaimResult<'a> {
     /// Can't claim the query because it is running on an other thread.
     Running(Running<'a>),
     /// Claiming the query results in a cycle.
-    Cycle,
+    Cycle { same_thread: bool },
     /// Successfully claimed the query.
     Claimed(ClaimGuard<'a>),
 }
@@ -62,7 +62,7 @@ impl SyncTable {
                     write,
                 ) {
                     BlockResult::Running(blocked_on) => ClaimResult::Running(blocked_on),
-                    BlockResult::Cycle => ClaimResult::Cycle,
+                    BlockResult::Cycle { same_thread } => ClaimResult::Cycle { same_thread },
                 }
             }
             std::collections::hash_map::Entry::Vacant(vacant_entry) => {

src/ingredient.rs

@@ -1,35 +1,50 @@
 use std::any::{Any, TypeId};
 use std::fmt;
 
+use crate::accumulator::accumulated_map::{AccumulatedMap, InputAccumulatedValues};
 use crate::cycle::{
-    empty_cycle_heads, CycleHeadKeys, CycleHeads, CycleRecoveryStrategy, IterationCount,
+    empty_cycle_heads, CycleHeads, CycleRecoveryStrategy, IterationCount, ProvisionalStatus,
-    ProvisionalStatus,
 };
-use crate::database::RawDatabase;
 use crate::function::VerifyResult;
+use crate::plumbing::IngredientIndices;
 use crate::runtime::Running;
 use crate::sync::Arc;
 use crate::table::memo::MemoTableTypes;
 use crate::table::Table;
 use crate::zalsa::{transmute_data_mut_ptr, transmute_data_ptr, IngredientIndex, Zalsa};
 use crate::zalsa_local::QueryOriginRef;
-use crate::{DatabaseKeyIndex, Id, Revision};
+use crate::{Database, DatabaseKeyIndex, Id, Revision};
 
 /// A "jar" is a group of ingredients that are added atomically.
-///
 /// Each type implementing jar can be added to the database at most once.
 pub trait Jar: Any {
-    /// Create the ingredients given the index of the first one.
+    /// This creates the ingredient dependencies of this jar. We need to split this from `create_ingredients()`
+    /// because while `create_ingredients()` is called, a lock on the ingredient map is held (to guarantee
+    /// atomicity), so other ingredients could not be created.
     ///
+    /// Only tracked fns use this.
+    fn create_dependencies(_zalsa: &Zalsa) -> IngredientIndices
+    where
+        Self: Sized,
+    {
+        IngredientIndices::empty()
+    }
+
+    /// Create the ingredients given the index of the first one.
     /// All subsequent ingredients will be assigned contiguous indices.
     fn create_ingredients(
-        zalsa: &mut Zalsa,
+        zalsa: &Zalsa,
         first_index: IngredientIndex,
-    ) -> Vec<Box<dyn Ingredient>>;
+        dependencies: IngredientIndices,
+    ) -> Vec<Box<dyn Ingredient>>
+    where
+        Self: Sized;
 
     /// This returns the [`TypeId`] of the ID struct, that is, the struct that wraps `salsa::Id`
     /// and carry the name of the jar.
-    fn id_struct_type_id() -> TypeId;
+    fn id_struct_type_id() -> TypeId
+    where
+        Self: Sized;
 }
 
 pub struct Location {
@@ -46,13 +61,12 @@ pub trait Ingredient: Any + std::fmt::Debug + Send + Sync {
     /// # Safety
     ///
     /// The passed in database needs to be the same one that the ingredient was created with.
-    unsafe fn maybe_changed_after(
+    unsafe fn maybe_changed_after<'db>(
-        &self,
+        &'db self,
-        zalsa: &crate::zalsa::Zalsa,
+        db: &'db dyn Database,
-        db: crate::database::RawDatabase<'_>,
         input: Id,
         revision: Revision,
-        cycle_heads: &mut CycleHeadKeys,
+        cycle_heads: &mut CycleHeads,
     ) -> VerifyResult;
 
     /// Returns information about the current provisional status of `input`.
@@ -137,9 +151,7 @@ pub trait Ingredient: Any + std::fmt::Debug + Send + Sync {
         );
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes>;
+    fn memo_table_types(&self) -> Arc<MemoTableTypes>;
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes>;
 
     fn fmt_index(&self, index: crate::Id, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt_index(self.debug_name(), index, fmt)
@@ -161,38 +173,26 @@ pub trait Ingredient: Any + std::fmt::Debug + Send + Sync {
 
     /// What values were accumulated during the creation of the value at `key_index`
     /// (if any).
-    ///
+    fn accumulated<'db>(
-    /// # Safety
-    ///
-    /// The passed in database needs to be the same one that the ingredient was created with.
-    #[cfg(feature = "accumulator")]
-    unsafe fn accumulated<'db>(
         &'db self,
-        db: RawDatabase<'db>,
+        db: &'db dyn Database,
         key_index: Id,
-    ) -> (
+    ) -> (Option<&'db AccumulatedMap>, InputAccumulatedValues) {
-        Option<&'db crate::accumulator::accumulated_map::AccumulatedMap>,
-        crate::accumulator::accumulated_map::InputAccumulatedValues,
-    ) {
         let _ = (db, key_index);
-        (
+        (None, InputAccumulatedValues::Empty)
-            None,
-            crate::accumulator::accumulated_map::InputAccumulatedValues::Empty,
-        )
     }
 
     /// Returns memory usage information about any instances of the ingredient,
     /// if applicable.
     #[cfg(feature = "salsa_unstable")]
-    fn memory_usage(&self, _db: &dyn crate::Database) -> Option<Vec<crate::database::SlotInfo>> {
+    fn memory_usage(&self, _db: &dyn Database) -> Option<Vec<crate::database::SlotInfo>> {
         None
     }
 }
 
 impl dyn Ingredient {
-    /// Equivalent to the `downcast` method on `Any`.
+    /// Equivalent to the `downcast` methods on `any`.
-    ///
+    /// Because we do not have dyn-upcasting support, we need this workaround.
-    /// Because we do not have dyn-downcasting support, we need this workaround.
     pub fn assert_type<T: Any>(&self) -> &T {
         assert_eq!(
             self.type_id(),
@@ -206,28 +206,8 @@ impl dyn Ingredient {
         unsafe { transmute_data_ptr(self) }
     }
 
-    /// Equivalent to the `downcast` methods on `Any`.
+    /// Equivalent to the `downcast` methods on `any`.
-    ///
+    /// Because we do not have dyn-upcasting support, we need this workaround.
-    /// Because we do not have dyn-downcasting support, we need this workaround.
-    ///
-    /// # Safety
-    ///
-    /// The contained value must be of type `T`.
-    pub unsafe fn assert_type_unchecked<T: Any>(&self) -> &T {
-        debug_assert_eq!(
-            self.type_id(),
-            TypeId::of::<T>(),
-            "ingredient `{self:?}` is not of type `{}`",
-            std::any::type_name::<T>()
-        );
-
-        // SAFETY: Guaranteed by caller.
-        unsafe { transmute_data_ptr(self) }
-    }
-
-    /// Equivalent to the `downcast` method on `Any`.
-    ///
-    /// Because we do not have dyn-downcasting support, we need this workaround.
     pub fn assert_type_mut<T: Any>(&mut self) -> &mut T {
         assert_eq!(
             Any::type_id(self),
@@ -250,11 +230,12 @@ pub(crate) fn fmt_index(debug_name: &str, id: Id, fmt: &mut fmt::Formatter<'_>)
 pub enum WaitForResult<'me> {
     Running(Running<'me>),
     Available,
-    Cycle,
+    Cycle { same_thread: bool },
 }
 
 impl WaitForResult<'_> {
-    pub const fn is_cycle(&self) -> bool {
+    /// Returns `true` if waiting for this input results in a cycle with another thread.
-        matches!(self, WaitForResult::Cycle)
+    pub const fn is_cycle_with_other_thread(&self) -> bool {
+        matches!(self, WaitForResult::Cycle { same_thread: false })
     }
 }

src/ingredient_cache.rs

@@ -1,237 +0,0 @@
-pub use imp::IngredientCache;
-
-#[cfg(feature = "inventory")]
-mod imp {
-    use crate::plumbing::Ingredient;
-    use crate::sync::atomic::{self, AtomicU32, Ordering};
-    use crate::zalsa::Zalsa;
-    use crate::IngredientIndex;
-
-    use std::marker::PhantomData;
-
-    /// Caches an ingredient index.
-    ///
-    /// Note that all ingredients are statically registered with `inventory`, so their
-    /// indices should be stable across any databases.
-    pub struct IngredientCache<I>
-    where
-        I: Ingredient,
-    {
-        ingredient_index: AtomicU32,
-        phantom: PhantomData<fn() -> I>,
-    }
-
-    impl<I> Default for IngredientCache<I>
-    where
-        I: Ingredient,
-    {
-        fn default() -> Self {
-            Self::new()
-        }
-    }
-
-    impl<I> IngredientCache<I>
-    where
-        I: Ingredient,
-    {
-        const UNINITIALIZED: u32 = u32::MAX;
-
-        /// Create a new cache
-        pub const fn new() -> Self {
-            Self {
-                ingredient_index: atomic::AtomicU32::new(Self::UNINITIALIZED),
-                phantom: PhantomData,
-            }
-        }
-
-        /// Get a reference to the ingredient in the database.
-        ///
-        /// If the ingredient index is not already in the cache, it will be loaded and cached.
-        ///
-        /// # Safety
-        ///
-        /// The `IngredientIndex` returned by the closure must reference a valid ingredient of
-        /// type `I` in the provided zalsa database.
-        pub unsafe fn get_or_create<'db>(
-            &self,
-            zalsa: &'db Zalsa,
-            load_index: impl Fn() -> IngredientIndex,
-        ) -> &'db I {
-            let mut ingredient_index = self.ingredient_index.load(Ordering::Acquire);
-            if ingredient_index == Self::UNINITIALIZED {
-                ingredient_index = self.get_or_create_index_slow(load_index).as_u32();
-            };
-
-            // SAFETY: `ingredient_index` is initialized from a valid `IngredientIndex`.
-            let ingredient_index = unsafe { IngredientIndex::new_unchecked(ingredient_index) };
-
-            // SAFETY: There are a two cases here:
-            // - The `create_index` closure was called due to the data being uncached. In this
-            //   case, the caller guarantees the index is in-bounds and has the correct type.
-            // - The index was cached. While the current database might not be the same database
-            //   the ingredient was initially loaded from, the `inventory` feature is enabled, so
-            //   ingredient indices are stable across databases. Thus the index is still in-bounds
-            //   and has the correct type.
-            unsafe {
-                zalsa
-                    .lookup_ingredient_unchecked(ingredient_index)
-                    .assert_type_unchecked()
-            }
-        }
-
-        #[cold]
-        #[inline(never)]
-        fn get_or_create_index_slow(
-            &self,
-            load_index: impl Fn() -> IngredientIndex,
-        ) -> IngredientIndex {
-            let ingredient_index = load_index();
-
-            // It doesn't matter if we overwrite any stores, as `create_index` should
-            // always return the same index when the `inventory` feature is enabled.
-            self.ingredient_index
-                .store(ingredient_index.as_u32(), Ordering::Release);
-
-            ingredient_index
-        }
-    }
-}
-
-#[cfg(not(feature = "inventory"))]
-mod imp {
-    use crate::nonce::Nonce;
-    use crate::plumbing::Ingredient;
-    use crate::sync::atomic::{AtomicU64, Ordering};
-    use crate::zalsa::{StorageNonce, Zalsa};
-    use crate::IngredientIndex;
-
-    use std::marker::PhantomData;
-    use std::mem;
-
-    /// Caches an ingredient index.
-    ///
-    /// With manual registration, ingredient indices can vary across databases,
-    /// but we can retain most of the benefit by optimizing for the the case of
-    /// a single database.
-    pub struct IngredientCache<I>
-    where
-        I: Ingredient,
-    {
-        // A packed representation of `Option<(Nonce<StorageNonce>, IngredientIndex)>`.
-        //
-        // This allows us to replace a lock in favor of an atomic load. This works thanks to `Nonce`
-        // having a niche, which means the entire type can fit into an `AtomicU64`.
-        cached_data: AtomicU64,
-        phantom: PhantomData<fn() -> I>,
-    }
-
-    impl<I> Default for IngredientCache<I>
-    where
-        I: Ingredient,
-    {
-        fn default() -> Self {
-            Self::new()
-        }
-    }
-
-    impl<I> IngredientCache<I>
-    where
-        I: Ingredient,
-    {
-        const UNINITIALIZED: u64 = 0;
-
-        /// Create a new cache
-        pub const fn new() -> Self {
-            Self {
-                cached_data: AtomicU64::new(Self::UNINITIALIZED),
-                phantom: PhantomData,
-            }
-        }
-
-        /// Get a reference to the ingredient in the database.
-        ///
-        /// If the ingredient is not already in the cache, it will be created.
-        ///
-        /// # Safety
-        ///
-        /// The `IngredientIndex` returned by the closure must reference a valid ingredient of
-        /// type `I` in the provided zalsa database.
-        #[inline(always)]
-        pub unsafe fn get_or_create<'db>(
-            &self,
-            zalsa: &'db Zalsa,
-            create_index: impl Fn() -> IngredientIndex,
-        ) -> &'db I {
-            let index = self.get_or_create_index(zalsa, create_index);
-
-            // SAFETY: There are a two cases here:
-            // - The `create_index` closure was called due to the data being uncached for the
-            //   provided database. In this case, the caller guarantees the index is in-bounds
-            //   and has the correct type.
-            // - We verified the index was cached for the same database, by the nonce check.
-            //   Thus the initial safety argument still applies.
-            unsafe {
-                zalsa
-                    .lookup_ingredient_unchecked(index)
-                    .assert_type_unchecked::<I>()
-            }
-        }
-
-        pub fn get_or_create_index(
-            &self,
-            zalsa: &Zalsa,
-            create_index: impl Fn() -> IngredientIndex,
-        ) -> IngredientIndex {
-            const _: () = assert!(
-                mem::size_of::<(Nonce<StorageNonce>, IngredientIndex)>() == mem::size_of::<u64>()
-            );
-
-            let cached_data = self.cached_data.load(Ordering::Acquire);
-            if cached_data == Self::UNINITIALIZED {
-                return self.get_or_create_index_slow(zalsa, create_index);
-            };
-
-            // Unpack our `u64` into the nonce and index.
-            //
-            // SAFETY: The lower bits of `cached_data` are initialized from a valid `IngredientIndex`.
-            let index = unsafe { IngredientIndex::new_unchecked(cached_data as u32) };
-
-            // SAFETY: We've checked against `UNINITIALIZED` (0) above and so the upper bits must be non-zero.
-            let nonce = crate::nonce::Nonce::<StorageNonce>::from_u32(unsafe {
-                std::num::NonZeroU32::new_unchecked((cached_data >> u32::BITS) as u32)
-            });
-
-            // The data was cached for a different database, we have to ensure the ingredient was
-            // created in ours.
-            if zalsa.nonce() != nonce {
-                return create_index();
-            }
-
-            index
-        }
-
-        #[cold]
-        #[inline(never)]
-        fn get_or_create_index_slow(
-            &self,
-            zalsa: &Zalsa,
-            create_index: impl Fn() -> IngredientIndex,
-        ) -> IngredientIndex {
-            let index = create_index();
-            let nonce = zalsa.nonce().into_u32().get() as u64;
-            let packed = (nonce << u32::BITS) | (index.as_u32() as u64);
-            debug_assert_ne!(packed, IngredientCache::<I>::UNINITIALIZED);
-
-            // Discard the result, whether we won over the cache or not doesn't matter.
-            _ = self.cached_data.compare_exchange(
-                IngredientCache::<I>::UNINITIALIZED,
-                packed,
-                Ordering::Release,
-                Ordering::Relaxed,
-            );
-
-            // Use our locally computed index regardless of which one was cached.
-            index
-        }
-    }
-}

src/input.rs

@@ -8,7 +8,7 @@ pub mod singleton;
 
 use input_field::FieldIngredientImpl;
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::function::VerifyResult;
 use crate::id::{AsId, FromId, FromIdWithDb};
 use crate::ingredient::Ingredient;
@@ -19,7 +19,7 @@ use crate::sync::Arc;
 use crate::table::memo::{MemoTable, MemoTableTypes};
 use crate::table::{Slot, Table};
 use crate::zalsa::{IngredientIndex, Zalsa};
-use crate::{zalsa_local, Durability, Id, Revision, Runtime};
+use crate::{Database, Durability, Id, Revision, Runtime};
 
 pub trait Configuration: Any {
     const DEBUG_NAME: &'static str;
@@ -40,11 +40,6 @@ pub trait Configuration: Any {
 
     /// A array of [`Durability`], one per each of the value fields.
     type Durabilities: Send + Sync + fmt::Debug + IndexMut<usize, Output = Durability>;
-
-    /// Returns the size of any heap allocations in the output value, in bytes.
-    fn heap_size(_value: &Self::Fields) -> Option<usize> {
-        None
-    }
 }
 
 pub struct JarImpl<C: Configuration> {
@@ -61,8 +56,9 @@ impl<C: Configuration> Default for JarImpl<C> {
 
 impl<C: Configuration> Jar for JarImpl<C> {
     fn create_ingredients(
-        _zalsa: &mut Zalsa,
+        _zalsa: &Zalsa,
         struct_index: crate::zalsa::IngredientIndex,
+        _dependencies: crate::memo_ingredient_indices::IngredientIndices,
     ) -> Vec<Box<dyn Ingredient>> {
         let struct_ingredient: IngredientImpl<C> = IngredientImpl::new(struct_index);
 
@@ -109,23 +105,20 @@ impl<C: Configuration> IngredientImpl<C> {
 
     pub fn new_input(
         &self,
-        zalsa: &Zalsa,
+        db: &dyn Database,
-        zalsa_local: &zalsa_local::ZalsaLocal,
         fields: C::Fields,
         revisions: C::Revisions,
         durabilities: C::Durabilities,
     ) -> C::Struct {
+        let (zalsa, zalsa_local) = db.zalsas();
+
         let id = self.singleton.with_scope(|| {
-            let (id, _) = zalsa_local.allocate(zalsa, self.ingredient_index, |_| Value::<C> {
+            zalsa_local.allocate(zalsa, self.ingredient_index, |_| Value::<C> {
                 fields,
                 revisions,
                 durabilities,
-                // SAFETY: We only ever access the memos of a value that we allocated through
+                memos: Default::default(),
-                // our `MemoTableTypes`.
+            })
-                memos: unsafe { MemoTable::new(self.memo_table_types()) },
-            });
-
-            id
         });
 
         FromIdWithDb::from_id(id, zalsa)
@@ -183,11 +176,11 @@ impl<C: Configuration> IngredientImpl<C> {
     /// The caller is responsible for selecting the appropriate element.
     pub fn field<'db>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn crate::Database,
-        zalsa_local: &'db zalsa_local::ZalsaLocal,
         id: C::Struct,
         field_index: usize,
     ) -> &'db C::Fields {
+        let (zalsa, zalsa_local) = db.zalsas();
         let field_ingredient_index = self.ingredient_index.successor(field_index);
         let id = id.as_id();
         let value = Self::data(zalsa, id);
@@ -203,13 +196,17 @@ impl<C: Configuration> IngredientImpl<C> {
 
     #[cfg(feature = "salsa_unstable")]
     /// Returns all data corresponding to the input struct.
-    pub fn entries<'db>(&'db self, zalsa: &'db Zalsa) -> impl Iterator<Item = &'db Value<C>> {
+    pub fn entries<'db>(
-        zalsa.table().slots_of::<Value<C>>()
+        &'db self,
+        db: &'db dyn crate::Database,
+    ) -> impl Iterator<Item = &'db Value<C>> {
+        db.zalsa().table().slots_of::<Value<C>>()
     }
 
     /// Peek at the field values without recording any read dependency.
     /// Used for debug printouts.
-    pub fn leak_fields<'db>(&'db self, zalsa: &'db Zalsa, id: C::Struct) -> &'db C::Fields {
+    pub fn leak_fields<'db>(&'db self, db: &'db dyn Database, id: C::Struct) -> &'db C::Fields {
+        let zalsa = db.zalsa();
         let id = id.as_id();
         let value = Self::data(zalsa, id);
         &value.fields
@@ -227,11 +224,10 @@ impl<C: Configuration> Ingredient for IngredientImpl<C> {
 
     unsafe fn maybe_changed_after(
         &self,
-        _zalsa: &crate::zalsa::Zalsa,
+        _db: &dyn Database,
-        _db: crate::database::RawDatabase<'_>,
         _input: Id,
         _revision: Revision,
-        _cycle_heads: &mut CycleHeadKeys,
+        _cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
         // Input ingredients are just a counter, they store no data, they are immortal.
         // Their *fields* are stored in function ingredients elsewhere.
@@ -242,19 +238,15 @@ impl<C: Configuration> Ingredient for IngredientImpl<C> {
         C::DEBUG_NAME
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        &self.memo_table_types
+        self.memo_table_types.clone()
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
-        &mut self.memo_table_types
     }
 
     /// Returns memory usage information about any inputs.
     #[cfg(feature = "salsa_unstable")]
-    fn memory_usage(&self, db: &dyn crate::Database) -> Option<Vec<crate::database::SlotInfo>> {
+    fn memory_usage(&self, db: &dyn Database) -> Option<Vec<crate::database::SlotInfo>> {
         let memory_usage = self
-            .entries(db.zalsa())
+            .entries(db)
             // SAFETY: The memo table belongs to a value that we allocated, so it
             // has the correct type.
             .map(|value| unsafe { value.memory_usage(&self.memo_table_types) })
@@ -312,7 +304,6 @@ where
     /// The `MemoTable` must belong to a `Value` of the correct type.
     #[cfg(feature = "salsa_unstable")]
     unsafe fn memory_usage(&self, memo_table_types: &MemoTableTypes) -> crate::database::SlotInfo {
-        let heap_size = C::heap_size(&self.fields);
         // SAFETY: The caller guarantees this is the correct types table.
         let memos = unsafe { memo_table_types.attach_memos(&self.memos) };
 
@@ -320,7 +311,6 @@ where
             debug_name: C::DEBUG_NAME,
             size_of_metadata: std::mem::size_of::<Self>() - std::mem::size_of::<C::Fields>(),
             size_of_fields: std::mem::size_of::<C::Fields>(),
-            heap_size_of_fields: heap_size,
             memos: memos.memory_usage(),
         }
     }

src/input/input_field.rs

@@ -1,14 +1,14 @@
 use std::fmt;
 use std::marker::PhantomData;
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::function::VerifyResult;
 use crate::ingredient::Ingredient;
 use crate::input::{Configuration, IngredientImpl, Value};
 use crate::sync::Arc;
 use crate::table::memo::MemoTableTypes;
 use crate::zalsa::IngredientIndex;
-use crate::{Id, Revision};
+use crate::{Database, Id, Revision};
 
 /// Ingredient used to represent the fields of a `#[salsa::input]`.
 ///
@@ -52,12 +52,12 @@ where
 
     unsafe fn maybe_changed_after(
         &self,
-        zalsa: &crate::zalsa::Zalsa,
+        db: &dyn Database,
-        _db: crate::database::RawDatabase<'_>,
         input: Id,
         revision: Revision,
-        _cycle_heads: &mut CycleHeadKeys,
+        _cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
+        let zalsa = db.zalsa();
         let value = <IngredientImpl<C>>::data(zalsa, input);
         VerifyResult::changed_if(value.revisions[self.field_index] > revision)
     }
@@ -76,11 +76,7 @@ where
         C::FIELD_DEBUG_NAMES[self.field_index]
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        unreachable!("input fields do not allocate pages")
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
         unreachable!("input fields do not allocate pages")
     }
 }

src/interned.rs

@@ -10,18 +10,18 @@ use crossbeam_utils::CachePadded;
 use intrusive_collections::{intrusive_adapter, LinkedList, LinkedListLink, UnsafeRef};
 use rustc_hash::FxBuildHasher;
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::durability::Durability;
 use crate::function::VerifyResult;
 use crate::id::{AsId, FromId};
 use crate::ingredient::Ingredient;
-use crate::plumbing::{Jar, ZalsaLocal};
+use crate::plumbing::{IngredientIndices, Jar, ZalsaLocal};
 use crate::revision::AtomicRevision;
 use crate::sync::{Arc, Mutex, OnceLock};
 use crate::table::memo::{MemoTable, MemoTableTypes, MemoTableWithTypesMut};
 use crate::table::Slot;
 use crate::zalsa::{IngredientIndex, Zalsa};
-use crate::{DatabaseKeyIndex, Event, EventKind, Id, Revision};
+use crate::{Database, DatabaseKeyIndex, Event, EventKind, Id, Revision};
 
 /// Trait that defines the key properties of an interned struct.
 ///
@@ -44,11 +44,6 @@ pub trait Configuration: Sized + 'static {
 
     /// The end user struct
     type Struct<'db>: Copy + FromId + AsId;
-
-    /// Returns the size of any heap allocations in the output value, in bytes.
-    fn heap_size(_value: &Self::Fields<'_>) -> Option<usize> {
-        None
-    }
 }
 
 pub trait InternedData: Sized + Eq + Hash + Clone + Sync + Send {}
@@ -204,7 +199,6 @@ where
     /// lock must be held for the shard containing the value.
     #[cfg(all(not(feature = "shuttle"), feature = "salsa_unstable"))]
     unsafe fn memory_usage(&self, memo_table_types: &MemoTableTypes) -> crate::database::SlotInfo {
-        let heap_size = C::heap_size(self.fields());
         // SAFETY: The caller guarantees we hold the lock for the shard containing the value, so we
         // have at-least read-only access to the value's memos.
         let memos = unsafe { &*self.memos.get() };
@@ -215,7 +209,6 @@ where
             debug_name: C::DEBUG_NAME,
             size_of_metadata: std::mem::size_of::<Self>() - std::mem::size_of::<C::Fields<'_>>(),
             size_of_fields: std::mem::size_of::<C::Fields<'_>>(),
-            heap_size_of_fields: heap_size,
             memos: memos.memory_usage(),
         }
     }
@@ -231,8 +224,9 @@ impl<C: Configuration> Default for JarImpl<C> {
 
 impl<C: Configuration> Jar for JarImpl<C> {
     fn create_ingredients(
-        _zalsa: &mut Zalsa,
+        _zalsa: &Zalsa,
         first_index: IngredientIndex,
+        _dependencies: IngredientIndices,
     ) -> Vec<Box<dyn Ingredient>> {
         vec![Box::new(IngredientImpl::<C>::new(first_index)) as _]
     }
@@ -303,8 +297,7 @@ where
     /// the database ends up trying to intern or allocate a new value.
     pub fn intern<'db, Key>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn crate::Database,
-        zalsa_local: &'db ZalsaLocal,
         key: Key,
         assemble: impl FnOnce(Id, Key) -> C::Fields<'db>,
     ) -> C::Struct<'db>
@@ -312,7 +305,7 @@ where
         Key: Hash,
         C::Fields<'db>: HashEqLike<Key>,
     {
-        FromId::from_id(self.intern_id(zalsa, zalsa_local, key, assemble))
+        FromId::from_id(self.intern_id(db, key, assemble))
     }
 
     /// Intern data to a unique reference.
@@ -327,8 +320,7 @@ where
     /// the database ends up trying to intern or allocate a new value.
     pub fn intern_id<'db, Key>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn crate::Database,
-        zalsa_local: &'db ZalsaLocal,
         key: Key,
         assemble: impl FnOnce(Id, Key) -> C::Fields<'db>,
     ) -> crate::Id
@@ -340,6 +332,8 @@ where
         // so instead we go with this and transmute the lifetime in the `eq` closure
         C::Fields<'db>: HashEqLike<Key>,
     {
+        let (zalsa, zalsa_local) = db.zalsas();
+
         // Record the current revision as active.
         let current_revision = zalsa.current_revision();
         self.revision_queue.record(current_revision);
@@ -422,6 +416,7 @@ where
         // Fill up the table for the first few revisions without attempting garbage collection.
         if !self.revision_queue.is_primed() {
             return self.intern_id_cold(
+                db,
                 key,
                 zalsa,
                 zalsa_local,
@@ -535,16 +530,16 @@ where
             // Insert the new value into the ID map.
             shard.key_map.insert_unique(hash, new_id, hasher);
 
+            // Free the memos associated with the previous interned value.
+            //
             // SAFETY: We hold the lock for the shard containing the value, and the
             // value has not been interned in the current revision, so no references to
             // it can exist.
-            let memo_table = unsafe { &mut *value.memos.get() };
+            let mut memo_table = unsafe { std::mem::take(&mut *value.memos.get()) };
 
-            // Free the memos associated with the previous interned value.
-            //
             // SAFETY: The memo table belongs to a value that we allocated, so it has the
             // correct type.
-            unsafe { self.clear_memos(zalsa, memo_table, new_id) };
+            unsafe { self.clear_memos(zalsa, &mut memo_table, new_id) };
 
             if value_shared.is_reusable::<C>() {
                 // Move the value to the front of the LRU list.
@@ -558,7 +553,16 @@ where
         }
 
         // If we could not find any stale slots, we are forced to allocate a new one.
-        self.intern_id_cold(key, zalsa, zalsa_local, assemble, shard, shard_index, hash)
+        self.intern_id_cold(
+            db,
+            key,
+            zalsa,
+            zalsa_local,
+            assemble,
+            shard,
+            shard_index,
+            hash,
+        )
     }
 
     /// The cold path for interning a value, allocating a new slot.
@@ -567,6 +571,7 @@ where
     #[allow(clippy::too_many_arguments)]
     fn intern_id_cold<'db, Key>(
         &'db self,
+        _db: &'db dyn crate::Database,
         key: Key,
         zalsa: &Zalsa,
         zalsa_local: &ZalsaLocal,
@@ -590,12 +595,10 @@ where
             .unwrap_or((Durability::MAX, Revision::max()));
 
         // Allocate the value slot.
-        let (id, value) = zalsa_local.allocate(zalsa, self.ingredient_index, |id| Value::<C> {
+        let id = zalsa_local.allocate(zalsa, self.ingredient_index, |id| Value::<C> {
             shard: shard_index as u16,
             link: LinkedListLink::new(),
-            // SAFETY: We only ever access the memos of a value that we allocated through
+            memos: UnsafeCell::new(MemoTable::default()),
-            // our `MemoTableTypes`.
-            memos: UnsafeCell::new(unsafe { MemoTable::new(self.memo_table_types()) }),
             // SAFETY: We call `from_internal_data` to restore the correct lifetime before access.
             fields: UnsafeCell::new(unsafe { self.to_internal_data(assemble(id, key)) }),
             shared: UnsafeCell::new(ValueShared {
@@ -605,6 +608,7 @@ where
             }),
         });
 
+        let value = zalsa.table().get::<Value<C>>(id);
         // SAFETY: We hold the lock for the shard containing the value.
         let value_shared = unsafe { &mut *value.shared.get() };
 
@@ -692,9 +696,6 @@ where
         };
 
         std::mem::forget(table_guard);
-
-        // Reset the table after having dropped any memos.
-        memo_table.reset();
     }
 
     // Hashes the value by its fields.
@@ -741,7 +742,8 @@ where
     }
 
     /// Lookup the data for an interned value based on its ID.
-    pub fn data<'db>(&'db self, zalsa: &'db Zalsa, id: Id) -> &'db C::Fields<'db> {
+    pub fn data<'db>(&'db self, db: &'db dyn Database, id: Id) -> &'db C::Fields<'db> {
+        let zalsa = db.zalsa();
         let value = zalsa.table().get::<Value<C>>(id);
 
         debug_assert!(
@@ -766,12 +768,12 @@ where
     /// Lookup the fields from an interned struct.
     ///
     /// Note that this is not "leaking" since no dependency edge is required.
-    pub fn fields<'db>(&'db self, zalsa: &'db Zalsa, s: C::Struct<'db>) -> &'db C::Fields<'db> {
+    pub fn fields<'db>(&'db self, db: &'db dyn Database, s: C::Struct<'db>) -> &'db C::Fields<'db> {
-        self.data(zalsa, AsId::as_id(&s))
+        self.data(db, AsId::as_id(&s))
     }
 
-    pub fn reset(&mut self, zalsa_mut: &mut Zalsa) {
+    pub fn reset(&mut self, db: &mut dyn Database) {
-        _ = zalsa_mut;
+        _ = db.zalsa_mut();
 
         for shard in self.shards.iter() {
             // We can clear the key maps now that we have cancelled all other handles.
@@ -781,8 +783,11 @@ where
 
     #[cfg(feature = "salsa_unstable")]
     /// Returns all data corresponding to the interned struct.
-    pub fn entries<'db>(&'db self, zalsa: &'db Zalsa) -> impl Iterator<Item = &'db Value<C>> {
+    pub fn entries<'db>(
-        zalsa.table().slots_of::<Value<C>>()
+        &'db self,
+        db: &'db dyn crate::Database,
+    ) -> impl Iterator<Item = &'db Value<C>> {
+        db.zalsa().table().slots_of::<Value<C>>()
     }
 }
 
@@ -800,12 +805,13 @@ where
 
     unsafe fn maybe_changed_after(
         &self,
-        zalsa: &crate::zalsa::Zalsa,
+        db: &dyn Database,
-        _db: crate::database::RawDatabase<'_>,
         input: Id,
         _revision: Revision,
-        _cycle_heads: &mut CycleHeadKeys,
+        _cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
+        let zalsa = db.zalsa();
+
         // Record the current revision as active.
         let current_revision = zalsa.current_revision();
         self.revision_queue.record(current_revision);
@@ -843,17 +849,13 @@ where
         C::DEBUG_NAME
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        &self.memo_table_types
+        self.memo_table_types.clone()
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
-        &mut self.memo_table_types
     }
 
     /// Returns memory usage information about any interned values.
     #[cfg(all(not(feature = "shuttle"), feature = "salsa_unstable"))]
-    fn memory_usage(&self, db: &dyn crate::Database) -> Option<Vec<crate::database::SlotInfo>> {
+    fn memory_usage(&self, db: &dyn Database) -> Option<Vec<crate::database::SlotInfo>> {
         use parking_lot::lock_api::RawMutex;
 
         for shard in self.shards.iter() {
@@ -862,7 +864,7 @@ where
         }
 
         let memory_usage = self
-            .entries(db.zalsa())
+            .entries(db)
             // SAFETY: The memo table belongs to a value that we allocated, so it
             // has the correct type. Additionally, we are holding the locks for all shards.
             .map(|value| unsafe { value.memory_usage(&self.memo_table_types) })

src/key.rs

@@ -1,9 +1,9 @@
 use core::fmt;
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::function::VerifyResult;
 use crate::zalsa::{IngredientIndex, Zalsa};
-use crate::Id;
+use crate::{Database, Id};
 
 // ANCHOR: DatabaseKeyIndex
 /// An integer that uniquely identifies a particular query instance within the
@@ -36,18 +36,16 @@ impl DatabaseKeyIndex {
 
     pub(crate) fn maybe_changed_after(
         &self,
-        db: crate::database::RawDatabase<'_>,
+        db: &dyn Database,
         zalsa: &Zalsa,
         last_verified_at: crate::Revision,
-        cycle_heads: &mut CycleHeadKeys,
+        cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
         // SAFETY: The `db` belongs to the ingredient
         unsafe {
-            // here, `db` has to be either the correct type already, or a subtype (as far as trait
-            // hierarchy is concerned)
             zalsa
                 .lookup_ingredient(self.ingredient_index())
-                .maybe_changed_after(zalsa, db, self.key_index(), last_verified_at, cycle_heads)
+                .maybe_changed_after(db, self.key_index(), last_verified_at, cycle_heads)
         }
     }
 

src/lib.rs

@@ -1,7 +1,6 @@
 #![deny(clippy::undocumented_unsafe_blocks)]
 #![forbid(unsafe_op_in_unsafe_fn)]
 
-#[cfg(feature = "accumulator")]
 mod accumulator;
 mod active_query;
 mod attach;
@@ -15,11 +14,13 @@ mod function;
 mod hash;
 mod id;
 mod ingredient;
-mod ingredient_cache;
 mod input;
 mod interned;
 mod key;
 mod memo_ingredient_indices;
+mod nonce;
+#[cfg(feature = "rayon")]
+mod parallel;
 mod return_mode;
 mod revision;
 mod runtime;
@@ -27,19 +28,12 @@ mod salsa_struct;
 mod storage;
 mod sync;
 mod table;
-mod tracing;
 mod tracked_struct;
 mod update;
 mod views;
 mod zalsa;
 mod zalsa_local;
 
-#[cfg(not(feature = "inventory"))]
-mod nonce;
-
-#[cfg(feature = "rayon")]
-mod parallel;
-
 #[cfg(feature = "rayon")]
 pub use parallel::{join, par_map};
 #[cfg(feature = "macros")]
@@ -48,12 +42,11 @@ pub use salsa_macros::{accumulator, db, input, interned, tracked, Supertype, Upd
 #[cfg(feature = "salsa_unstable")]
 pub use self::database::IngredientInfo;
 
-#[cfg(feature = "accumulator")]
 pub use self::accumulator::Accumulator;
 pub use self::active_query::Backtrace;
 pub use self::cancelled::Cancelled;
 pub use self::cycle::CycleRecoveryAction;
-pub use self::database::Database;
+pub use self::database::{AsDynDatabase, Database};
 pub use self::database_impl::DatabaseImpl;
 pub use self::durability::Durability;
 pub use self::event::{Event, EventKind};
@@ -70,9 +63,7 @@ pub use self::zalsa::IngredientIndex;
 pub use crate::attach::{attach, with_attached_database};
 
 pub mod prelude {
-    #[cfg(feature = "accumulator")]
+    pub use crate::{Accumulator, Database, Setter};
-    pub use crate::accumulator::Accumulator;
-    pub use crate::{Database, Setter};
 }
 
 /// Internal names used by salsa macros.
@@ -85,16 +76,13 @@ pub mod plumbing {
     pub use std::any::TypeId;
     pub use std::option::Option::{self, None, Some};
 
-    #[cfg(feature = "accumulator")]
-    pub use salsa_macro_rules::setup_accumulator_impl;
     pub use salsa_macro_rules::{
-        gate_accumulated, macro_if, maybe_backdate, maybe_default, maybe_default_tt,
+        macro_if, maybe_backdate, maybe_default, maybe_default_tt, return_mode_expression,
-        return_mode_expression, return_mode_ty, setup_input_struct, setup_interned_struct,
+        return_mode_ty, setup_accumulator_impl, setup_input_struct, setup_interned_struct,
         setup_tracked_assoc_fn_body, setup_tracked_fn, setup_tracked_method_body,
         setup_tracked_struct, unexpected_cycle_initial, unexpected_cycle_recovery,
     };
 
-    #[cfg(feature = "accumulator")]
     pub use crate::accumulator::Accumulator;
     pub use crate::attach::{attach, with_attached_database};
     pub use crate::cycle::{CycleRecoveryAction, CycleRecoveryStrategy};
@@ -102,7 +90,6 @@ pub mod plumbing {
     pub use crate::durability::Durability;
     pub use crate::id::{AsId, FromId, FromIdWithDb, Id};
     pub use crate::ingredient::{Ingredient, Jar, Location};
-    pub use crate::ingredient_cache::IngredientCache;
     pub use crate::key::DatabaseKeyIndex;
     pub use crate::memo_ingredient_indices::{
         IngredientIndices, MemoIngredientIndices, MemoIngredientMap, MemoIngredientSingletonIndex,
@@ -112,18 +99,14 @@ pub mod plumbing {
     pub use crate::runtime::{stamp, Runtime, Stamp};
     pub use crate::salsa_struct::SalsaStructInDb;
     pub use crate::storage::{HasStorage, Storage};
-    pub use crate::table::memo::MemoTableWithTypes;
     pub use crate::tracked_struct::TrackedStructInDb;
     pub use crate::update::helper::{Dispatch as UpdateDispatch, Fallback as UpdateFallback};
     pub use crate::update::{always_update, Update};
-    pub use crate::views::DatabaseDownCaster;
     pub use crate::zalsa::{
-        register_jar, transmute_data_ptr, views, ErasedJar, HasJar, IngredientIndex, JarKind,
+        transmute_data_ptr, views, IngredientCache, IngredientIndex, Zalsa, ZalsaDatabase,
-        Zalsa, ZalsaDatabase,
     };
     pub use crate::zalsa_local::ZalsaLocal;
 
-    #[cfg(feature = "accumulator")]
     pub mod accumulator {
         pub use crate::accumulator::{IngredientImpl, JarImpl};
     }

src/memo_ingredient_indices.rs

@@ -49,11 +49,11 @@ pub trait NewMemoIngredientIndices {
     ///
     /// The memo types must be correct.
     unsafe fn create(
-        zalsa: &mut Zalsa,
+        zalsa: &Zalsa,
         struct_indices: IngredientIndices,
         ingredient: IngredientIndex,
         memo_type: MemoEntryType,
-        intern_ingredient_memo_types: Option<&mut Arc<MemoTableTypes>>,
+        intern_ingredient_memo_types: Option<Arc<MemoTableTypes>>,
     ) -> Self;
 }
 
@@ -62,39 +62,34 @@ impl NewMemoIngredientIndices for MemoIngredientIndices {
     ///
     /// The memo types must be correct.
     unsafe fn create(
-        zalsa: &mut Zalsa,
+        zalsa: &Zalsa,
         struct_indices: IngredientIndices,
         ingredient: IngredientIndex,
         memo_type: MemoEntryType,
-        _intern_ingredient_memo_types: Option<&mut Arc<MemoTableTypes>>,
+        _intern_ingredient_memo_types: Option<Arc<MemoTableTypes>>,
     ) -> Self {
         debug_assert!(
             _intern_ingredient_memo_types.is_none(),
             "intern ingredient can only have a singleton memo ingredient"
         );
-
         let Some(&last) = struct_indices.indices.last() else {
             unreachable!("Attempting to construct struct memo mapping for non tracked function?")
         };
-
         let mut indices = Vec::new();
         indices.resize(
             (last.as_u32() as usize) + 1,
             MemoIngredientIndex::from_usize((u32::MAX - 1) as usize),
         );
-
         for &struct_ingredient in &struct_indices.indices {
-            let memo_ingredient_index =
+            let memo_types = zalsa
-                zalsa.next_memo_ingredient_index(struct_ingredient, ingredient);
+                .lookup_ingredient(struct_ingredient)
-            indices[struct_ingredient.as_u32() as usize] = memo_ingredient_index;
+                .memo_table_types();
 
-            let (struct_ingredient, _) = zalsa.lookup_ingredient_mut(struct_ingredient);
+            let mi = zalsa.next_memo_ingredient_index(struct_ingredient, ingredient);
-            let memo_types = Arc::get_mut(struct_ingredient.memo_table_types_mut())
+            memo_types.set(mi, &memo_type);
-                .expect("memo tables are not shared until database initialization is complete");
 
-            memo_types.set(memo_ingredient_index, memo_type);
+            indices[struct_ingredient.as_u32() as usize] = mi;
         }
-
         MemoIngredientIndices {
             indices: indices.into_boxed_slice(),
         }
@@ -151,27 +146,25 @@ impl MemoIngredientMap for MemoIngredientSingletonIndex {
 impl NewMemoIngredientIndices for MemoIngredientSingletonIndex {
     #[inline]
     unsafe fn create(
-        zalsa: &mut Zalsa,
+        zalsa: &Zalsa,
         indices: IngredientIndices,
         ingredient: IngredientIndex,
         memo_type: MemoEntryType,
-        intern_ingredient_memo_types: Option<&mut Arc<MemoTableTypes>>,
+        intern_ingredient_memo_types: Option<Arc<MemoTableTypes>>,
     ) -> Self {
         let &[struct_ingredient] = &*indices.indices else {
             unreachable!("Attempting to construct struct memo mapping from enum?")
         };
 
-        let memo_ingredient_index = zalsa.next_memo_ingredient_index(struct_ingredient, ingredient);
         let memo_types = intern_ingredient_memo_types.unwrap_or_else(|| {
-            let (struct_ingredient, _) = zalsa.lookup_ingredient_mut(struct_ingredient);
+            zalsa
-            struct_ingredient.memo_table_types_mut()
+                .lookup_ingredient(struct_ingredient)
+                .memo_table_types()
         });
 
-        Arc::get_mut(memo_types)
+        let mi = zalsa.next_memo_ingredient_index(struct_ingredient, ingredient);
-            .expect("memo tables are not shared until database initialization is complete")
+        memo_types.set(mi, &memo_type);
-            .set(memo_ingredient_index, memo_type);
+        Self(mi)
-
-        Self(memo_ingredient_index)
     }
 }
 

src/parallel.rs

@@ -1,91 +1,44 @@
 use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator};
 
-use crate::{database::RawDatabase, views::DatabaseDownCaster, Database};
+use crate::Database;
 
 pub fn par_map<Db, F, T, R, C>(db: &Db, inputs: impl IntoParallelIterator<Item = T>, op: F) -> C
 where
-    Db: Database + ?Sized + Send,
+    Db: Database + ?Sized,
     F: Fn(&Db, T) -> R + Sync + Send,
     T: Send,
     R: Send + Sync,
     C: FromParallelIterator<R>,
 {
-    let views = db.zalsa().views();
-    let caster = &views.downcaster_for::<Db>();
-    let db_caster = &views.downcaster_for::<dyn Database>();
     inputs
         .into_par_iter()
-        .map_with(
+        .map_with(DbForkOnClone(db.fork_db()), |db, element| {
-            DbForkOnClone(db.fork_db(), caster, db_caster),
+            op(db.0.as_view(), element)
-            |db, element| op(db.as_view(), element),
+        })
-        )
         .collect()
 }
 
-struct DbForkOnClone<'views, Db: Database + ?Sized>(
+struct DbForkOnClone(Box<dyn Database>);
-    RawDatabase<'static>,
-    &'views DatabaseDownCaster<Db>,
-    &'views DatabaseDownCaster<dyn Database>,
-);
 
-// SAFETY: `T: Send` -> `&own T: Send`, `DbForkOnClone` is an owning pointer
+impl Clone for DbForkOnClone {
-unsafe impl<Db: Send + Database + ?Sized> Send for DbForkOnClone<'_, Db> {}
-
-impl<Db: Database + ?Sized> DbForkOnClone<'_, Db> {
-    fn as_view(&self) -> &Db {
-        // SAFETY: The downcaster ensures that the pointer is valid for the lifetime of the view.
-        unsafe { self.1.downcast_unchecked(self.0) }
-    }
-}
-
-impl<Db: Database + ?Sized> Drop for DbForkOnClone<'_, Db> {
-    fn drop(&mut self) {
-        // SAFETY: `caster` is derived from a `db` fitting for our database clone
-        let db = unsafe { self.1.downcast_mut_unchecked(self.0) };
-        // SAFETY: `db` has been box allocated and leaked by `fork_db`
-        _ = unsafe { Box::from_raw(db) };
-    }
-}
-
-impl<Db: Database + ?Sized> Clone for DbForkOnClone<'_, Db> {
     fn clone(&self) -> Self {
-        DbForkOnClone(
+        DbForkOnClone(self.0.fork_db())
-            // SAFETY: `caster` is derived from a `db` fitting for our database clone
-            unsafe { self.2.downcast_unchecked(self.0) }.fork_db(),
-            self.1,
-            self.2,
-        )
     }
 }
 
-pub fn join<A, B, RA, RB, Db: Send + Database + ?Sized>(db: &Db, a: A, b: B) -> (RA, RB)
+pub fn join<A, B, RA, RB, Db: Database + ?Sized>(db: &Db, a: A, b: B) -> (RA, RB)
 where
     A: FnOnce(&Db) -> RA + Send,
     B: FnOnce(&Db) -> RB + Send,
     RA: Send,
     RB: Send,
 {
-    #[derive(Copy, Clone)]
-    struct AssertSend<T>(T);
-    // SAFETY: We send owning pointers over, which are Send, given the `Db` type parameter above is Send
-    unsafe impl<T> Send for AssertSend<T> {}
-
-    let caster = &db.zalsa().views().downcaster_for::<Db>();
     // we need to fork eagerly, as `rayon::join_context` gives us no option to tell whether we get
     // moved to another thread before the closure is executed
-    let db_a = AssertSend(db.fork_db());
+    let db_a = db.fork_db();
-    let db_b = AssertSend(db.fork_db());
+    let db_b = db.fork_db();
-    let res = rayon::join(
+    rayon::join(
-        // SAFETY: `caster` is derived from a `db` fitting for our database clone
+        move || a(db_a.as_view::<Db>()),
-        move || a(unsafe { caster.downcast_unchecked({ db_a }.0) }),
+        move || b(db_b.as_view::<Db>()),
-        // SAFETY: `caster` is derived from a `db` fitting for our database clone
+    )
-        move || b(unsafe { caster.downcast_unchecked({ db_b }.0) }),
-    );
-
-    // SAFETY: `db` has been box allocated and leaked by `fork_db`
-    // FIXME: Clean this mess up, RAII
-    _ = unsafe { Box::from_raw(caster.downcast_mut_unchecked(db_a.0)) };
-    // SAFETY: `db` has been box allocated and leaked by `fork_db`
-    _ = unsafe { Box::from_raw(caster.downcast_mut_unchecked(db_b.0)) };
-    res
 }

src/runtime.rs

@@ -51,7 +51,7 @@ pub(crate) enum BlockResult<'me> {
     ///
     /// The lock is hold by the current thread or there's another thread that is waiting on the current thread,
     /// and blocking this thread on the other thread would result in a deadlock/cycle.
-    Cycle,
+    Cycle { same_thread: bool },
 }
 
 pub struct Running<'me>(Box<BlockedOnInner<'me>>);
@@ -86,7 +86,7 @@ impl Running<'_> {
             })
         });
 
-        crate::tracing::debug!(
+        tracing::debug!(
             "block_on: thread {thread_id:?} is blocking on {database_key:?} in thread {other_id:?}",
         );
 
@@ -180,7 +180,7 @@ impl Runtime {
     }
 
     pub(crate) fn set_cancellation_flag(&self) {
-        crate::tracing::trace!("set_cancellation_flag");
+        tracing::trace!("set_cancellation_flag");
         self.revision_canceled.store(true, Ordering::Release);
     }
 
@@ -206,7 +206,7 @@ impl Runtime {
         let r_old = self.current_revision();
         let r_new = r_old.next();
         self.revisions[0] = r_new;
-        crate::tracing::debug!("new_revision: {r_old:?} -> {r_new:?}");
+        tracing::debug!("new_revision: {r_old:?} -> {r_new:?}");
         r_new
     }
 
@@ -230,14 +230,14 @@ impl Runtime {
         let thread_id = thread::current().id();
         // Cycle in the same thread.
         if thread_id == other_id {
-            return BlockResult::Cycle;
+            return BlockResult::Cycle { same_thread: true };
         }
 
         let dg = self.dependency_graph.lock();
 
         if dg.depends_on(other_id, thread_id) {
-            crate::tracing::debug!("block_on: cycle detected for {database_key:?} in thread {thread_id:?} on {other_id:?}");
+            tracing::debug!("block_on: cycle detected for {database_key:?} in thread {thread_id:?} on {other_id:?}");
-            return BlockResult::Cycle;
+            return BlockResult::Cycle { same_thread: false };
         }
 
         BlockResult::Running(Running(Box::new(BlockedOnInner {

src/salsa_struct.rs

@@ -1,9 +1,8 @@
 use std::any::TypeId;
 
 use crate::memo_ingredient_indices::{IngredientIndices, MemoIngredientMap};
-use crate::table::memo::MemoTableWithTypes;
 use crate::zalsa::Zalsa;
-use crate::{Id, Revision};
+use crate::Id;
 
 pub trait SalsaStructInDb: Sized {
     type MemoIngredientMap: MemoIngredientMap;
@@ -17,7 +16,7 @@ pub trait SalsaStructInDb: Sized {
     /// While implementors of this trait may call [`crate::zalsa::JarEntry::get_or_create`]
     /// to create the ingredient, they aren't required to. For example, supertypes recursively
     /// call [`crate::zalsa::JarEntry::get_or_create`] for their variants and combine them.
-    fn lookup_ingredient_index(zalsa: &Zalsa) -> IngredientIndices;
+    fn lookup_or_create_ingredient_index(zalsa: &Zalsa) -> IngredientIndices;
 
     /// Plumbing to support nested salsa supertypes.
     ///
@@ -63,16 +62,4 @@ pub trait SalsaStructInDb: Sized {
     /// Why `TypeId` and not `IngredientIndex`? Because it's cheaper and easier: the `TypeId` is readily
     /// available at compile time, while the `IngredientIndex` requires a runtime lookup.
     fn cast(id: Id, type_id: TypeId) -> Option<Self>;
-
-    /// Return the memo table associated with `id`.
-    ///
-    /// # Safety
-    ///
-    /// The parameter `current_revision` must be the current revision of the owner of database
-    /// owning this table.
-    unsafe fn memo_table(
-        zalsa: &Zalsa,
-        id: Id,
-        current_revision: Revision,
-    ) -> MemoTableWithTypes<'_>;
 }

src/storage.rs

@@ -2,9 +2,8 @@
 use std::marker::PhantomData;
 use std::panic::RefUnwindSafe;
 
-use crate::database::RawDatabase;
 use crate::sync::{Arc, Condvar, Mutex};
-use crate::zalsa::{ErasedJar, HasJar, Zalsa, ZalsaDatabase};
+use crate::zalsa::{Zalsa, ZalsaDatabase};
 use crate::zalsa_local::{self, ZalsaLocal};
 use crate::{Database, Event, EventKind};
 
@@ -43,15 +42,8 @@ impl<Db: Database> Default for StorageHandle<Db> {
 
 impl<Db: Database> StorageHandle<Db> {
     pub fn new(event_callback: Option<Box<dyn Fn(crate::Event) + Send + Sync + 'static>>) -> Self {
-        Self::with_jars(event_callback, Vec::new())
-    }
-
-    fn with_jars(
-        event_callback: Option<Box<dyn Fn(crate::Event) + Send + Sync + 'static>>,
-        jars: Vec<ErasedJar>,
-    ) -> Self {
         Self {
-            zalsa_impl: Arc::new(Zalsa::new::<Db>(event_callback, jars)),
+            zalsa_impl: Arc::new(Zalsa::new::<Db>(event_callback)),
             coordinate: CoordinateDrop(Arc::new(Coordinate {
                 clones: Mutex::new(1),
                 cvar: Default::default(),
@@ -123,11 +115,6 @@ impl<Db: Database> Storage<Db> {
         }
     }
 
-    /// Returns a builder for database storage.
-    pub fn builder() -> StorageBuilder<Db> {
-        StorageBuilder::default()
-    }
-
     /// Convert this instance of [`Storage`] into a [`StorageHandle`].
     ///
     /// This will discard the local state of this [`Storage`], thereby returning a value that
@@ -181,54 +168,6 @@ impl<Db: Database> Storage<Db> {
     // ANCHOR_END: cancel_other_workers
 }
 
-/// A builder for a [`Storage`] instance.
-///
-/// This type can be created with the [`Storage::builder`] function.
-pub struct StorageBuilder<Db> {
-    jars: Vec<ErasedJar>,
-    event_callback: Option<Box<dyn Fn(crate::Event) + Send + Sync + 'static>>,
-    _db: PhantomData<Db>,
-}
-
-impl<Db> Default for StorageBuilder<Db> {
-    fn default() -> Self {
-        Self {
-            jars: Vec::new(),
-            event_callback: None,
-            _db: PhantomData,
-        }
-    }
-}
-
-impl<Db: Database> StorageBuilder<Db> {
-    /// Set a callback for salsa events.
-    ///
-    /// The `event_callback` function will be invoked by the salsa runtime at various points during execution.
-    pub fn event_callback(
-        mut self,
-        callback: Box<dyn Fn(crate::Event) + Send + Sync + 'static>,
-    ) -> Self {
-        self.event_callback = Some(callback);
-        self
-    }
-
-    /// Manually register an ingredient.
-    ///
-    /// Manual ingredient registration is necessary when the `inventory` feature is disabled.
-    pub fn ingredient<I: HasJar>(mut self) -> Self {
-        self.jars.push(ErasedJar::erase::<I>());
-        self
-    }
-
-    /// Construct the [`Storage`] using the provided builder options.
-    pub fn build(self) -> Storage<Db> {
-        Storage {
-            handle: StorageHandle::with_jars(self.event_callback, self.jars),
-            zalsa_local: ZalsaLocal::new(),
-        }
-    }
-}
-
 #[allow(clippy::undocumented_unsafe_blocks)] // TODO(#697) document safety
 unsafe impl<T: HasStorage> ZalsaDatabase for T {
     #[inline(always)]
@@ -246,8 +185,8 @@ unsafe impl<T: HasStorage> ZalsaDatabase for T {
     }
 
     #[inline(always)]
-    fn fork_db(&self) -> RawDatabase<'static> {
+    fn fork_db(&self) -> Box<dyn Database> {
-        Box::leak(Box::new(self.clone())).into()
+        Box::new(self.clone())
     }
 }
 

src/sync.rs

@@ -5,6 +5,40 @@ pub mod shim {
     pub use shuttle::sync::*;
     pub use shuttle::{thread, thread_local};
 
+    pub mod papaya {
+        use std::hash::{BuildHasher, Hash};
+        use std::marker::PhantomData;
+
+        pub struct HashMap<K, V, S>(super::Mutex<std::collections::HashMap<K, V, S>>);
+
+        impl<K, V, S: Default> Default for HashMap<K, V, S> {
+            fn default() -> Self {
+                Self(super::Mutex::default())
+            }
+        }
+
+        pub struct LocalGuard<'a>(PhantomData<&'a ()>);
+
+        impl<K, V, S> HashMap<K, V, S>
+        where
+            K: Eq + Hash,
+            V: Clone,
+            S: BuildHasher,
+        {
+            pub fn guard(&self) -> LocalGuard<'_> {
+                LocalGuard(PhantomData)
+            }
+
+            pub fn get(&self, key: &K, _guard: &LocalGuard<'_>) -> Option<V> {
+                self.0.lock().get(key).cloned()
+            }
+
+            pub fn insert(&self, key: K, value: V, _guard: &LocalGuard<'_>) {
+                self.0.lock().insert(key, value);
+            }
+        }
+    }
+
     /// A wrapper around shuttle's `Mutex` to mirror parking-lot's API.
     #[derive(Default, Debug)]
     pub struct Mutex<T>(shuttle::sync::Mutex<T>);
@@ -23,6 +57,24 @@ pub mod shim {
         }
     }
 
+    /// A wrapper around shuttle's `RwLock` to mirror parking-lot's API.
+    #[derive(Default, Debug)]
+    pub struct RwLock<T>(shuttle::sync::RwLock<T>);
+
+    impl<T> RwLock<T> {
+        pub fn read(&self) -> RwLockReadGuard<'_, T> {
+            self.0.read().unwrap()
+        }
+
+        pub fn write(&self) -> RwLockWriteGuard<'_, T> {
+            self.0.write().unwrap()
+        }
+
+        pub fn get_mut(&mut self) -> &mut T {
+            self.0.get_mut().unwrap()
+        }
+    }
+
     /// A wrapper around shuttle's `Condvar` to mirror parking-lot's API.
     #[derive(Default, Debug)]
     pub struct Condvar(shuttle::sync::Condvar);
@@ -112,7 +164,7 @@ pub mod shim {
 
 #[cfg(not(feature = "shuttle"))]
 pub mod shim {
-    pub use parking_lot::{Mutex, MutexGuard};
+    pub use parking_lot::{Mutex, MutexGuard, RwLock};
     pub use std::sync::*;
     pub use std::{thread, thread_local};
 
@@ -121,6 +173,48 @@ pub mod shim {
         pub use std::sync::atomic::*;
     }
 
+    pub mod papaya {
+        use std::hash::{BuildHasher, Hash};
+
+        pub use papaya::LocalGuard;
+
+        pub struct HashMap<K, V, S>(papaya::HashMap<K, V, S>);
+
+        impl<K, V, S: Default> Default for HashMap<K, V, S> {
+            fn default() -> Self {
+                Self(
+                    papaya::HashMap::builder()
+                        .capacity(256) // A relatively large capacity to hopefully avoid resizing.
+                        .resize_mode(papaya::ResizeMode::Blocking)
+                        .hasher(S::default())
+                        .build(),
+                )
+            }
+        }
+
+        impl<K, V, S> HashMap<K, V, S>
+        where
+            K: Eq + Hash,
+            V: Clone,
+            S: BuildHasher,
+        {
+            #[inline]
+            pub fn guard(&self) -> LocalGuard<'_> {
+                self.0.guard()
+            }
+
+            #[inline]
+            pub fn get(&self, key: &K, guard: &LocalGuard<'_>) -> Option<V> {
+                self.0.get(key, guard).cloned()
+            }
+
+            #[inline]
+            pub fn insert(&self, key: K, value: V, guard: &LocalGuard<'_>) {
+                self.0.insert(key, value, guard);
+            }
+        }
+    }
+
     /// A wrapper around parking-lot's `Condvar` to mirror shuttle's API.
     pub struct Condvar(parking_lot::Condvar);
 

src/table.rs

@@ -34,7 +34,7 @@ pub struct Table {
 ///
 /// Implementors of this trait need to make sure that their type is unique with respect to
 /// their owning ingredient as the allocation strategy relies on this.
-pub unsafe trait Slot: Any + Send + Sync {
+pub(crate) unsafe trait Slot: Any + Send + Sync {
     /// Access the [`MemoTable`][] for this slot.
     ///
     /// # Safety condition
@@ -220,42 +220,17 @@ impl Table {
         PageIndex::new(self.pages.push(Page::new::<T>(ingredient, memo_types)))
     }
 
-    /// Get the memo table associated with `id` for the concrete type `T`.
+    /// Get the memo table associated with `id`
     ///
-    /// # Safety
+    /// # Safety condition
     ///
-    /// The parameter `current_revision` must be the current revision of the database
+    /// The parameter `current_revision` MUST be the current revision
-    /// owning this table.
+    /// of the owner of database owning this table.
-    ///
+    pub(crate) unsafe fn memos(
-    /// # Panics
-    ///
-    /// If `page` is out of bounds or the type `T` is incorrect.
-    pub unsafe fn memos<T: Slot>(
         &self,
         id: Id,
         current_revision: Revision,
     ) -> MemoTableWithTypes<'_> {
-        let (page, slot) = split_id(id);
-        let page = self.pages[page.0].assert_type::<T>();
-        let slot = &page.data()[slot.0];
-
-        // SAFETY: The caller is required to pass the `current_revision`.
-        let memos = unsafe { slot.memos(current_revision) };
-
-        // SAFETY: The `Page` keeps the correct memo types.
-        unsafe { page.0.memo_types.attach_memos(memos) }
-    }
-
-    /// Get the memo table associated with `id`.
-    ///
-    /// Unlike `Table::memos`, this does not require a concrete type, and instead uses dynamic
-    /// dispatch.
-    ///
-    /// # Safety
-    ///
-    /// The parameter `current_revision` must be the current revision of the owner of database
-    /// owning this table.
-    pub unsafe fn dyn_memos(&self, id: Id, current_revision: Revision) -> MemoTableWithTypes<'_> {
         let (page, slot) = split_id(id);
         let page = &self.pages[page.0];
         // SAFETY: We supply a proper slot pointer and the caller is required to pass the `current_revision`.
@@ -286,8 +261,6 @@ impl Table {
             .flat_map(|view| view.data())
     }
 
-    #[cold]
-    #[inline(never)]
     pub(crate) fn fetch_or_push_page<T: Slot>(
         &self,
         ingredient: IngredientIndex,
@@ -301,7 +274,6 @@ impl Table {
         {
             return page;
         }
-
         self.push_page::<T>(ingredient, memo_types())
     }
 
@@ -314,23 +286,22 @@ impl Table {
     }
 }
 
-impl<'db, T: Slot> PageView<'db, T> {
+impl<'p, T: Slot> PageView<'p, T> {
     #[inline]
-    fn page_data(&self) -> &'db [PageDataEntry<T>] {
+    fn page_data(&self) -> &'p [PageDataEntry<T>] {
         let len = self.0.allocated.load(Ordering::Acquire);
         // SAFETY: `len` is the initialized length of the page
         unsafe { slice::from_raw_parts(self.0.data.cast::<PageDataEntry<T>>().as_ptr(), len) }
     }
 
     #[inline]
-    fn data(&self) -> &'db [T] {
+    fn data(&self) -> &'p [T] {
         let len = self.0.allocated.load(Ordering::Acquire);
         // SAFETY: `len` is the initialized length of the page
         unsafe { slice::from_raw_parts(self.0.data.cast::<T>().as_ptr(), len) }
     }
 
-    #[inline]
+    pub(crate) fn allocate<V>(&self, page: PageIndex, value: V) -> Result<Id, V>
-    pub(crate) fn allocate<V>(&self, page: PageIndex, value: V) -> Result<(Id, &'db T), V>
     where
         V: FnOnce(Id) -> T,
     {
@@ -351,14 +322,11 @@ impl<'db, T: Slot> PageView<'db, T> {
         // interior
         unsafe { (*entry.get()).write(value(id)) };
 
-        // SAFETY: We just initialized the value above.
-        let value = unsafe { (*entry.get()).assume_init_ref() };
-
         // Update the length (this must be done after initialization as otherwise an uninitialized
         // read could occur!)
         self.0.allocated.store(index + 1, Ordering::Release);
 
-        Ok((id, value))
+        Ok(id)
     }
 }
 
@@ -405,7 +373,6 @@ impl Page {
             slot.0 < len,
             "out of bounds access `{slot:?}` (maximum slot `{len}`)"
         );
-
         // SAFETY: We have checked that the resulting pointer will be within bounds.
         unsafe {
             self.data
@@ -416,10 +383,13 @@ impl Page {
 
     #[inline]
     fn assert_type<T: Slot>(&self) -> PageView<'_, T> {
-        if self.slot_type_id != TypeId::of::<T>() {
+        assert_eq!(
-            type_assert_failed::<T>(self);
+            self.slot_type_id,
-        }
+            TypeId::of::<T>(),
-
+            "page has slot type `{:?}` but `{:?}` was expected",
+            self.slot_type_name,
+            std::any::type_name::<T>(),
+        );
         PageView(self, PhantomData)
     }
 
@@ -433,17 +403,6 @@ impl Page {
     }
 }
 
-/// This function is explicitly outlined to avoid debug machinery in the hot-path.
-#[cold]
-#[inline(never)]
-fn type_assert_failed<T: 'static>(page: &Page) -> ! {
-    panic!(
-        "page has slot type `{:?}` but `{:?}` was expected",
-        page.slot_type_name,
-        std::any::type_name::<T>(),
-    )
-}
-
 impl Drop for Page {
     fn drop(&mut self) {
         let len = *self.allocated.get_mut();

src/table/memo.rs

@@ -3,39 +3,19 @@ use std::fmt::Debug;
 use std::mem;
 use std::ptr::{self, NonNull};
 
+use portable_atomic::hint::spin_loop;
+use thin_vec::ThinVec;
+
 use crate::sync::atomic::{AtomicPtr, Ordering};
+use crate::sync::{OnceLock, RwLock};
 use crate::{zalsa::MemoIngredientIndex, zalsa_local::QueryOriginRef};
 
 /// The "memo table" stores the memoized results of tracked function calls.
 /// Every tracked function must take a salsa struct as its first argument
 /// and memo tables are attached to those salsa structs as auxiliary data.
-pub struct MemoTable {
+#[derive(Default)]
-    memos: Box<[MemoEntry]>,
+pub(crate) struct MemoTable {
-}
+    memos: RwLock<ThinVec<MemoEntry>>,
-
-impl MemoTable {
-    /// Create a `MemoTable` with slots for memos from the provided `MemoTableTypes`.
-    ///
-    /// # Safety
-    ///
-    /// The created memo table must only be accessed with the same `MemoTableTypes`.
-    pub unsafe fn new(types: &MemoTableTypes) -> Self {
-        // Note that the safety invariant guarantees that any indices in-bounds for
-        // this table are also in-bounds for its `MemoTableTypes`, as `MemoTableTypes`
-        // is append-only.
-        Self {
-            memos: (0..types.len()).map(|_| MemoEntry::default()).collect(),
-        }
-    }
-
-    /// Reset any memos in the table.
-    ///
-    /// Note that the memo entries should be freed manually before calling this function.
-    pub fn reset(&mut self) {
-        for memo in &mut self.memos {
-            *memo = MemoEntry::default();
-        }
-    }
 }
 
 pub trait Memo: Any + Send + Sync {
@@ -70,8 +50,13 @@ struct MemoEntry {
     atomic_memo: AtomicPtr<DummyMemo>,
 }
 
-#[derive(Clone, Copy, Debug)]
+#[derive(Default)]
 pub struct MemoEntryType {
+    data: OnceLock<MemoEntryTypeData>,
+}
+
+#[derive(Clone, Copy, Debug)]
+struct MemoEntryTypeData {
     /// The `type_id` of the erased memo type `M`
     type_id: TypeId,
 
@@ -104,10 +89,17 @@ impl MemoEntryType {
     #[inline]
     pub fn of<M: Memo>() -> Self {
         Self {
-            type_id: TypeId::of::<M>(),
+            data: OnceLock::from(MemoEntryTypeData {
-            to_dyn_fn: Self::to_dyn_fn::<M>(),
+                type_id: TypeId::of::<M>(),
+                to_dyn_fn: Self::to_dyn_fn::<M>(),
+            }),
         }
     }
+
+    #[inline]
+    fn load(&self) -> Option<&MemoEntryTypeData> {
+        self.data.get()
+    }
 }
 
 /// Dummy placeholder type that we use when erasing the memo type `M` in [`MemoEntryData`][].
@@ -127,7 +119,6 @@ impl Memo for DummyMemo {
                 debug_name: "dummy",
                 size_of_metadata: 0,
                 size_of_fields: 0,
-                heap_size_of_fields: None,
                 memos: Vec::new(),
             },
         }
@@ -136,21 +127,43 @@ impl Memo for DummyMemo {
 
 #[derive(Default)]
 pub struct MemoTableTypes {
-    types: Vec<MemoEntryType>,
+    types: boxcar::Vec<MemoEntryType>,
 }
 
 impl MemoTableTypes {
     pub(crate) fn set(
-        &mut self,
+        &self,
         memo_ingredient_index: MemoIngredientIndex,
-        memo_type: MemoEntryType,
+        memo_type: &MemoEntryType,
     ) {
-        self.types
+        let memo_ingredient_index = memo_ingredient_index.as_usize();
-            .insert(memo_ingredient_index.as_usize(), memo_type);
-    }
 
-    pub fn len(&self) -> usize {
+        // Try to create our entry if it has not already been created.
-        self.types.len()
+        if memo_ingredient_index >= self.types.count() {
+            while self.types.push(MemoEntryType::default()) < memo_ingredient_index {}
+        }
+
+        loop {
+            let Some(memo_entry_type) = self.types.get(memo_ingredient_index) else {
+                // It's possible that someone else began pushing to our index but has not
+                // completed the entry's initialization yet, as `boxcar` is lock-free. This
+                // is extremely unlikely given initialization is just a handful of instructions.
+                // Additionally, this function is generally only called on startup, so we can
+                // just spin here.
+                spin_loop();
+                continue;
+            };
+
+            memo_entry_type
+                .data
+                .set(
+                    *memo_type.data.get().expect(
+                        "cannot provide an empty `MemoEntryType` for `MemoEntryType::set()`",
+                    ),
+                )
+                .expect("memo type should only be set once");
+            break;
+        }
     }
 
     /// # Safety
@@ -176,7 +189,7 @@ impl MemoTableTypes {
     }
 }
 
-pub struct MemoTableWithTypes<'a> {
+pub(crate) struct MemoTableWithTypes<'a> {
     types: &'a MemoTableTypes,
     memos: &'a MemoTable,
 }
@@ -187,62 +200,97 @@ impl MemoTableWithTypes<'_> {
         memo_ingredient_index: MemoIngredientIndex,
         memo: NonNull<M>,
     ) -> Option<NonNull<M>> {
-        let MemoEntry { atomic_memo } = self.memos.memos.get(memo_ingredient_index.as_usize())?;
+        // The type must already exist, we insert it when creating the memo ingredient.
-
+        assert_eq!(
-        // SAFETY: Any indices that are in-bounds for the `MemoTable` are also in-bounds for its
-        // corresponding `MemoTableTypes`, by construction.
-        let type_ = unsafe {
             self.types
                 .types
-                .get_unchecked(memo_ingredient_index.as_usize())
+                .get(memo_ingredient_index.as_usize())
-        };
+                .and_then(MemoEntryType::load)?
+                .type_id,
+            TypeId::of::<M>(),
+            "inconsistent type-id for `{memo_ingredient_index:?}`"
+        );
 
-        // Verify that the we are casting to the correct type.
+        // If the memo slot is already occupied, it must already have the
-        if type_.type_id != TypeId::of::<M>() {
+        // right type info etc, and we only need the read-lock.
-            type_assert_failed(memo_ingredient_index);
+        if let Some(MemoEntry { atomic_memo }) = self
+            .memos
+            .memos
+            .read()
+            .get(memo_ingredient_index.as_usize())
+        {
+            let old_memo =
+                atomic_memo.swap(MemoEntryType::to_dummy(memo).as_ptr(), Ordering::AcqRel);
+
+            let old_memo = NonNull::new(old_memo);
+
+            // SAFETY: `type_id` check asserted above
+            return old_memo.map(|old_memo| unsafe { MemoEntryType::from_dummy(old_memo) });
         }
 
-        let old_memo = atomic_memo.swap(MemoEntryType::to_dummy(memo).as_ptr(), Ordering::AcqRel);
+        // Otherwise we need the write lock.
-
+        self.insert_cold(memo_ingredient_index, memo)
-        // SAFETY: We asserted that the type is correct above.
+    }
-        NonNull::new(old_memo).map(|old_memo| unsafe { MemoEntryType::from_dummy(old_memo) })
+
+    #[cold]
+    fn insert_cold<M: Memo>(
+        self,
+        memo_ingredient_index: MemoIngredientIndex,
+        memo: NonNull<M>,
+    ) -> Option<NonNull<M>> {
+        let memo_ingredient_index = memo_ingredient_index.as_usize();
+        let mut memos = self.memos.memos.write();
+
+        // Grow the table if needed.
+        if memos.len() <= memo_ingredient_index {
+            let additional_len = memo_ingredient_index - memos.len() + 1;
+            memos.reserve(additional_len);
+            while memos.len() <= memo_ingredient_index {
+                memos.push(MemoEntry::default());
+            }
+        }
+
+        let old_entry = mem::replace(
+            memos[memo_ingredient_index].atomic_memo.get_mut(),
+            MemoEntryType::to_dummy(memo).as_ptr(),
+        );
+
+        // SAFETY: The `TypeId` is asserted in `insert()`.
+        NonNull::new(old_entry).map(|memo| unsafe { MemoEntryType::from_dummy(memo) })
     }
 
-    /// Returns a pointer to the memo at the given index, if one has been inserted.
     #[inline]
     pub(crate) fn get<M: Memo>(
         self,
         memo_ingredient_index: MemoIngredientIndex,
     ) -> Option<NonNull<M>> {
-        let MemoEntry { atomic_memo } = self.memos.memos.get(memo_ingredient_index.as_usize())?;
+        let read = self.memos.memos.read();
-
+        let memo = read.get(memo_ingredient_index.as_usize())?;
-        // SAFETY: Any indices that are in-bounds for the `MemoTable` are also in-bounds for its
+        let type_ = self
-        // corresponding `MemoTableTypes`, by construction.
+            .types
-        let type_ = unsafe {
+            .types
-            self.types
+            .get(memo_ingredient_index.as_usize())
-                .types
+            .and_then(MemoEntryType::load)?;
-                .get_unchecked(memo_ingredient_index.as_usize())
+        assert_eq!(
-        };
+            type_.type_id,
-
+            TypeId::of::<M>(),
-        // Verify that the we are casting to the correct type.
+            "inconsistent type-id for `{memo_ingredient_index:?}`"
-        if type_.type_id != TypeId::of::<M>() {
+        );
-            type_assert_failed(memo_ingredient_index);
+        let memo = NonNull::new(memo.atomic_memo.load(Ordering::Acquire))?;
-        }
+        // SAFETY: `type_id` check asserted above
-
+        Some(unsafe { MemoEntryType::from_dummy(memo) })
-        NonNull::new(atomic_memo.load(Ordering::Acquire))
-            // SAFETY: We asserted that the type is correct above.
-            .map(|memo| unsafe { MemoEntryType::from_dummy(memo) })
     }
 
     #[cfg(feature = "salsa_unstable")]
     pub(crate) fn memory_usage(&self) -> Vec<crate::database::MemoInfo> {
         let mut memory_usage = Vec::new();
-        for (index, memo) in self.memos.memos.iter().enumerate() {
+        let memos = self.memos.memos.read();
+        for (index, memo) in memos.iter().enumerate() {
             let Some(memo) = NonNull::new(memo.atomic_memo.load(Ordering::Acquire)) else {
                 continue;
             };
 
-            let Some(type_) = self.types.types.get(index) else {
+            let Some(type_) = self.types.types.get(index).and_then(MemoEntryType::load) else {
                 continue;
             };
 
@@ -269,30 +317,32 @@ impl MemoTableWithTypesMut<'_> {
         memo_ingredient_index: MemoIngredientIndex,
         f: impl FnOnce(&mut M),
     ) {
-        let Some(MemoEntry { atomic_memo }) =
+        let Some(type_) = self
-            self.memos.memos.get_mut(memo_ingredient_index.as_usize())
+            .types
+            .types
+            .get(memo_ingredient_index.as_usize())
+            .and_then(MemoEntryType::load)
         else {
             return;
         };
+        assert_eq!(
+            type_.type_id,
+            TypeId::of::<M>(),
+            "inconsistent type-id for `{memo_ingredient_index:?}`"
+        );
 
-        // SAFETY: Any indices that are in-bounds for the `MemoTable` are also in-bounds for its
+        // If the memo slot is already occupied, it must already have the
-        // corresponding `MemoTableTypes`, by construction.
+        // right type info etc, and we only need the read-lock.
-        let type_ = unsafe {
+        let memos = self.memos.memos.get_mut();
-            self.types
+        let Some(MemoEntry { atomic_memo }) = memos.get_mut(memo_ingredient_index.as_usize())
-                .types
+        else {
-                .get_unchecked(memo_ingredient_index.as_usize())
+            return;
         };
-
-        // Verify that the we are casting to the correct type.
-        if type_.type_id != TypeId::of::<M>() {
-            type_assert_failed(memo_ingredient_index);
-        }
-
         let Some(memo) = NonNull::new(*atomic_memo.get_mut()) else {
             return;
         };
 
-        // SAFETY: We asserted that the type is correct above.
+        // SAFETY: `type_id` check asserted above
         f(unsafe { MemoEntryType::from_dummy(memo).as_mut() });
     }
 
@@ -307,7 +357,7 @@ impl MemoTableWithTypesMut<'_> {
     #[inline]
     pub unsafe fn drop(&mut self) {
         let types = self.types.types.iter();
-        for (type_, memo) in std::iter::zip(types, &mut self.memos.memos) {
+        for ((_, type_), memo) in std::iter::zip(types, self.memos.memos.get_mut()) {
             // SAFETY: The types match as per our constructor invariant.
             unsafe { memo.take(type_) };
         }
@@ -321,12 +371,12 @@ impl MemoTableWithTypesMut<'_> {
         &mut self,
         mut f: impl FnMut(MemoIngredientIndex, Box<dyn Memo>),
     ) {
-        self.memos
+        let memos = self.memos.memos.get_mut();
-            .memos
+        memos
             .iter_mut()
             .zip(self.types.types.iter())
             .enumerate()
-            .filter_map(|(index, (memo, type_))| {
+            .filter_map(|(index, (memo, (_, type_)))| {
                 // SAFETY: The types match as per our constructor invariant.
                 let memo = unsafe { memo.take(type_)? };
                 Some((MemoIngredientIndex::from_usize(index), memo))
@@ -335,13 +385,6 @@ impl MemoTableWithTypesMut<'_> {
     }
 }
 
-/// This function is explicitly outlined to avoid debug machinery in the hot-path.
-#[cold]
-#[inline(never)]
-fn type_assert_failed(memo_ingredient_index: MemoIngredientIndex) -> ! {
-    panic!("inconsistent type-id for `{memo_ingredient_index:?}`")
-}
-
 impl MemoEntry {
     /// # Safety
     ///
@@ -350,6 +393,7 @@ impl MemoEntry {
     unsafe fn take(&mut self, type_: &MemoEntryType) -> Option<Box<dyn Memo>> {
         let memo = mem::replace(self.atomic_memo.get_mut(), ptr::null_mut());
         let memo = NonNull::new(memo)?;
+        let type_ = type_.load()?;
         // SAFETY: Our preconditions.
         Some(unsafe { Box::from_raw((type_.to_dyn_fn)(memo).as_ptr()) })
     }

src/tracing.rs

@@ -1,54 +0,0 @@
-//! Wrappers around `tracing` macros that avoid inlining debug machinery into the hot path,
-//! as tracing events are typically only enabled for debugging purposes.
-
-macro_rules! trace {
-    ($($x:tt)*) => {
-        crate::tracing::event!(TRACE, $($x)*)
-    };
-}
-
-macro_rules! info {
-    ($($x:tt)*) => {
-        crate::tracing::event!(INFO, $($x)*)
-    };
-}
-
-macro_rules! debug {
-    ($($x:tt)*) => {
-        crate::tracing::event!(DEBUG, $($x)*)
-    };
-}
-
-macro_rules! debug_span {
-    ($($x:tt)*) => {
-        crate::tracing::span!(DEBUG, $($x)*)
-    };
-}
-
-macro_rules! event {
-    ($level:ident, $($x:tt)*) => {{
-        let event = {
-            #[cold] #[inline(never)] || { ::tracing::event!(::tracing::Level::$level, $($x)*) }
-        };
-
-        if ::tracing::enabled!(::tracing::Level::$level) {
-            event();
-        }
-    }};
-}
-
-macro_rules! span {
-    ($level:ident, $($x:tt)*) => {{
-        let span = {
-            #[cold] #[inline(never)] || { ::tracing::span!(::tracing::Level::$level, $($x)*) }
-        };
-
-        if ::tracing::enabled!(::tracing::Level::$level) {
-            span()
-        } else {
-            ::tracing::Span::none()
-        }
-    }};
-}
-
-pub(crate) use {debug, debug_span, event, info, span, trace};

src/tracked_struct.rs

@@ -10,7 +10,7 @@ use crossbeam_queue::SegQueue;
 use thin_vec::ThinVec;
 use tracked_field::FieldIngredientImpl;
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::function::VerifyResult;
 use crate::id::{AsId, FromId};
 use crate::ingredient::{Ingredient, Jar};
@@ -23,7 +23,7 @@ use crate::sync::Arc;
 use crate::table::memo::{MemoTable, MemoTableTypes, MemoTableWithTypesMut};
 use crate::table::{Slot, Table};
 use crate::zalsa::{IngredientIndex, Zalsa};
-use crate::{Durability, Event, EventKind, Id, Revision};
+use crate::{Database, Durability, Event, EventKind, Id, Revision};
 
 pub mod tracked_field;
 
@@ -90,11 +90,6 @@ pub trait Configuration: Sized + 'static {
         old_fields: *mut Self::Fields<'db>,
         new_fields: Self::Fields<'db>,
     ) -> bool;
-
-    /// Returns the size of any heap allocations in the output value, in bytes.
-    fn heap_size(_value: &Self::Fields<'_>) -> Option<usize> {
-        None
-    }
 }
 // ANCHOR_END: Configuration
 
@@ -115,8 +110,9 @@ impl<C: Configuration> Default for JarImpl<C> {
 
 impl<C: Configuration> Jar for JarImpl<C> {
     fn create_ingredients(
-        _zalsa: &mut Zalsa,
+        _zalsa: &Zalsa,
         struct_index: crate::zalsa::IngredientIndex,
+        _dependencies: crate::memo_ingredient_indices::IngredientIndices,
     ) -> Vec<Box<dyn Ingredient>> {
         let struct_ingredient = <IngredientImpl<C>>::new(struct_index);
 
@@ -380,10 +376,11 @@ where
 
     pub fn new_struct<'db>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn Database,
-        zalsa_local: &'db ZalsaLocal,
         mut fields: C::Fields<'db>,
     ) -> C::Struct<'db> {
+        let (zalsa, zalsa_local) = db.zalsas();
+
         let identity_hash = IdentityHash {
             ingredient_index: self.ingredient_index,
             hash: crate::hash::hash(&C::untracked_fields(&fields)),
@@ -401,7 +398,7 @@ where
         if let Some(id) = zalsa_local.tracked_struct_id(&identity) {
             // The struct already exists in the intern map.
             let index = self.database_key_index(id);
-            crate::tracing::trace!("Reuse tracked struct {id:?}", id = index);
+            tracing::trace!("Reuse tracked struct {id:?}", id = index);
             zalsa_local.add_output(index);
 
             // SAFETY: The `id` was present in the interned map, so the value must be initialized.
@@ -427,7 +424,7 @@ where
         // in the struct map.
         let id = self.allocate(zalsa, zalsa_local, current_revision, &current_deps, fields);
         let key = self.database_key_index(id);
-        crate::tracing::trace!("Allocated new tracked struct {key:?}");
+        tracing::trace!("Allocated new tracked struct {key:?}");
         zalsa_local.add_output(key);
         zalsa_local.store_tracked_struct_id(identity, id);
         FromId::from_id(id)
@@ -447,9 +444,7 @@ where
             // lifetime erase for storage
             fields: unsafe { mem::transmute::<C::Fields<'db>, C::Fields<'static>>(fields) },
             revisions: C::new_revisions(current_deps.changed_at),
-            // SAFETY: We only ever access the memos of a value that we allocated through
+            memos: Default::default(),
-            // our `MemoTableTypes`.
-            memos: unsafe { MemoTable::new(self.memo_table_types()) },
         };
 
         while let Some(id) = self.free_list.pop() {
@@ -459,7 +454,7 @@ where
             // If the generation would overflow, we are forced to leak the slot. Note that this
             // shouldn't be a problem in general as sufficient bits are reserved for the generation.
             let Some(id) = id.next_generation() else {
-                crate::tracing::info!(
+                tracing::info!(
                     "leaking tracked struct {:?} due to generation overflow",
                     self.database_key_index(id)
                 );
@@ -482,9 +477,7 @@ where
             return id;
         }
 
-        let (id, _) = zalsa_local.allocate::<Value<C>>(zalsa, self.ingredient_index, value);
+        zalsa_local.allocate::<Value<C>>(zalsa, self.ingredient_index, value)
-
-        id
     }
 
     /// Get mutable access to the data for `id` -- this holds a write lock for the duration
@@ -561,7 +554,7 @@ where
             // the unlikely case that the ID is already at its maximum generation, we are forced to leak
             // the previous slot and allocate a new value.
             if id.generation() == u32::MAX {
-                crate::tracing::info!(
+                tracing::info!(
                     "leaking tracked struct {:?} due to generation overflow",
                     self.database_key_index(id)
                 );
@@ -608,11 +601,11 @@ where
             // Note that we hold the lock and have exclusive access to the tracked struct data,
             // so there should be no live instances of IDs from the previous generation. We clear
             // the memos and return a new ID here as if we have allocated a new slot.
-            let memo_table = data.memo_table_mut();
+            let mut table = data.take_memo_table();
 
             // SAFETY: The memo table belongs to a value that we allocated, so it has the
             // correct type.
-            unsafe { self.clear_memos(zalsa, memo_table, id) };
+            unsafe { self.clear_memos(zalsa, &mut table, id) };
 
             id = id
                 .next_generation()
@@ -681,11 +674,11 @@ where
 
         // SAFETY: We have acquired the write lock
         let data = unsafe { &mut *data_raw };
-        let memo_table = data.memo_table_mut();
+        let mut memo_table = data.take_memo_table();
 
         // SAFETY: The memo table belongs to a value that we allocated, so it
         // has the correct type.
-        unsafe { self.clear_memos(zalsa, memo_table, id) };
+        unsafe { self.clear_memos(zalsa, &mut memo_table, id) };
 
         // now that all cleanup has occurred, make available for re-use
         self.free_list.push(id);
@@ -731,20 +724,17 @@ where
         };
 
         mem::forget(table_guard);
-
-        // Reset the table after having dropped any memos.
-        memo_table.reset();
     }
 
     /// Return reference to the field data ignoring dependency tracking.
     /// Used for debugging.
     pub fn leak_fields<'db>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn Database,
         s: C::Struct<'db>,
     ) -> &'db C::Fields<'db> {
         let id = AsId::as_id(&s);
-        let data = Self::data(zalsa.table(), id);
+        let data = Self::data(db.zalsa().table(), id);
         data.fields()
     }
 
@@ -754,11 +744,11 @@ where
     /// The caller is responsible for selecting the appropriate element.
     pub fn tracked_field<'db>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn crate::Database,
-        zalsa_local: &'db ZalsaLocal,
         s: C::Struct<'db>,
         relative_tracked_index: usize,
     ) -> &'db C::Fields<'db> {
+        let (zalsa, zalsa_local) = db.zalsas();
         let id = AsId::as_id(&s);
         let field_ingredient_index = self.ingredient_index.successor(relative_tracked_index);
         let data = Self::data(zalsa.table(), id);
@@ -782,9 +772,10 @@ where
     /// The caller is responsible for selecting the appropriate element.
     pub fn untracked_field<'db>(
         &'db self,
-        zalsa: &'db Zalsa,
+        db: &'db dyn crate::Database,
         s: C::Struct<'db>,
     ) -> &'db C::Fields<'db> {
+        let zalsa = db.zalsa();
         let id = AsId::as_id(&s);
         let data = Self::data(zalsa.table(), id);
 
@@ -799,8 +790,11 @@ where
 
     #[cfg(feature = "salsa_unstable")]
     /// Returns all data corresponding to the tracked struct.
-    pub fn entries<'db>(&'db self, zalsa: &'db Zalsa) -> impl Iterator<Item = &'db Value<C>> {
+    pub fn entries<'db>(
-        zalsa.table().slots_of::<Value<C>>()
+        &'db self,
+        db: &'db dyn crate::Database,
+    ) -> impl Iterator<Item = &'db Value<C>> {
+        db.zalsa().table().slots_of::<Value<C>>()
     }
 }
 
@@ -818,11 +812,10 @@ where
 
     unsafe fn maybe_changed_after(
         &self,
-        _zalsa: &crate::zalsa::Zalsa,
+        _db: &dyn Database,
-        _db: crate::database::RawDatabase<'_>,
         _input: Id,
         _revision: Revision,
-        _cycle_heads: &mut CycleHeadKeys,
+        _cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
         // Any change to a tracked struct results in a new ID generation.
         VerifyResult::unchanged()
@@ -856,19 +849,15 @@ where
         C::DEBUG_NAME
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        &self.memo_table_types
+        self.memo_table_types.clone()
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
-        &mut self.memo_table_types
     }
 
     /// Returns memory usage information about any tracked structs.
     #[cfg(feature = "salsa_unstable")]
-    fn memory_usage(&self, db: &dyn crate::Database) -> Option<Vec<crate::database::SlotInfo>> {
+    fn memory_usage(&self, db: &dyn Database) -> Option<Vec<crate::database::SlotInfo>> {
         let memory_usage = self
-            .entries(db.zalsa())
+            .entries(db)
             // SAFETY: The memo table belongs to a value that we allocated, so it
             // has the correct type.
             .map(|value| unsafe { value.memory_usage(&self.memo_table_types) })
@@ -902,12 +891,13 @@ where
         unsafe { mem::transmute::<&C::Fields<'static>, &C::Fields<'_>>(&self.fields) }
     }
 
-    fn memo_table_mut(&mut self) -> &mut MemoTable {
+    fn take_memo_table(&mut self) -> MemoTable {
         // This fn is only called after `updated_at` has been set to `None`;
         // this ensures that there is no concurrent access
         // (and that the `&mut self` is accurate...).
         assert!(self.updated_at.load().is_none());
-        &mut self.memos
+
+        mem::take(&mut self.memos)
     }
 
     fn read_lock(&self, current_revision: Revision) {
@@ -940,7 +930,6 @@ where
     /// The `MemoTable` must belong to a `Value` of the correct type.
     #[cfg(feature = "salsa_unstable")]
     unsafe fn memory_usage(&self, memo_table_types: &MemoTableTypes) -> crate::database::SlotInfo {
-        let heap_size = C::heap_size(self.fields());
         // SAFETY: The caller guarantees this is the correct types table.
         let memos = unsafe { memo_table_types.attach_memos(&self.memos) };
 
@@ -948,7 +937,6 @@ where
             debug_name: C::DEBUG_NAME,
             size_of_metadata: mem::size_of::<Self>() - mem::size_of::<C::Fields<'_>>(),
             size_of_fields: mem::size_of::<C::Fields<'_>>(),
-            heap_size_of_fields: heap_size,
             memos: memos.memory_usage(),
         }
     }
@@ -961,8 +949,9 @@ where
 {
     #[inline(always)]
     unsafe fn memos(&self, current_revision: Revision) -> &crate::table::memo::MemoTable {
-        // Acquiring the read lock here with the current revision to ensure that there
+        // Acquiring the read lock here with the current revision
-        // is no danger of a race when deleting a tracked struct.
+        // ensures that there is no danger of a race
+        // when deleting a tracked struct.
         self.read_lock(current_revision);
         &self.memos
     }
@@ -982,19 +971,19 @@ mod tests {
         let mut d = DisambiguatorMap::default();
         // set up all 4 permutations of differing field values
         let h1 = IdentityHash {
-            ingredient_index: IngredientIndex::new(0),
+            ingredient_index: IngredientIndex::from(0),
             hash: 0,
         };
         let h2 = IdentityHash {
-            ingredient_index: IngredientIndex::new(1),
+            ingredient_index: IngredientIndex::from(1),
             hash: 0,
         };
         let h3 = IdentityHash {
-            ingredient_index: IngredientIndex::new(0),
+            ingredient_index: IngredientIndex::from(0),
             hash: 1,
         };
         let h4 = IdentityHash {
-            ingredient_index: IngredientIndex::new(1),
+            ingredient_index: IngredientIndex::from(1),
             hash: 1,
         };
         assert_eq!(d.disambiguate(h1), Disambiguator(0));
@@ -1012,42 +1001,42 @@ mod tests {
         let mut d = IdentityMap::default();
         // set up all 8 permutations of differing field values
         let i1 = Identity {
-            ingredient_index: IngredientIndex::new(0),
+            ingredient_index: IngredientIndex::from(0),
             hash: 0,
             disambiguator: Disambiguator(0),
         };
         let i2 = Identity {
-            ingredient_index: IngredientIndex::new(1),
+            ingredient_index: IngredientIndex::from(1),
             hash: 0,
             disambiguator: Disambiguator(0),
         };
         let i3 = Identity {
-            ingredient_index: IngredientIndex::new(0),
+            ingredient_index: IngredientIndex::from(0),
             hash: 1,
             disambiguator: Disambiguator(0),
         };
         let i4 = Identity {
-            ingredient_index: IngredientIndex::new(1),
+            ingredient_index: IngredientIndex::from(1),
             hash: 1,
             disambiguator: Disambiguator(0),
         };
         let i5 = Identity {
-            ingredient_index: IngredientIndex::new(0),
+            ingredient_index: IngredientIndex::from(0),
             hash: 0,
             disambiguator: Disambiguator(1),
         };
         let i6 = Identity {
-            ingredient_index: IngredientIndex::new(1),
+            ingredient_index: IngredientIndex::from(1),
             hash: 0,
             disambiguator: Disambiguator(1),
         };
         let i7 = Identity {
-            ingredient_index: IngredientIndex::new(0),
+            ingredient_index: IngredientIndex::from(0),
             hash: 1,
             disambiguator: Disambiguator(1),
         };
         let i8 = Identity {
-            ingredient_index: IngredientIndex::new(1),
+            ingredient_index: IngredientIndex::from(1),
             hash: 1,
             disambiguator: Disambiguator(1),
         };

src/tracked_struct/tracked_field.rs

@@ -1,13 +1,13 @@
 use std::marker::PhantomData;
 
-use crate::cycle::CycleHeadKeys;
+use crate::cycle::CycleHeads;
 use crate::function::VerifyResult;
 use crate::ingredient::Ingredient;
 use crate::sync::Arc;
 use crate::table::memo::MemoTableTypes;
 use crate::tracked_struct::{Configuration, Value};
 use crate::zalsa::IngredientIndex;
-use crate::Id;
+use crate::{Database, Id};
 
 /// Created for each tracked struct.
 ///
@@ -55,14 +55,14 @@ where
         self.ingredient_index
     }
 
-    unsafe fn maybe_changed_after(
+    unsafe fn maybe_changed_after<'db>(
-        &self,
+        &'db self,
-        zalsa: &crate::zalsa::Zalsa,
+        db: &'db dyn Database,
-        _db: crate::database::RawDatabase<'_>,
         input: Id,
         revision: crate::Revision,
-        _cycle_heads: &mut CycleHeadKeys,
+        _cycle_heads: &mut CycleHeads,
     ) -> VerifyResult {
+        let zalsa = db.zalsa();
         let data = <super::IngredientImpl<C>>::data(zalsa.table(), input);
         let field_changed_at = data.revisions[self.field_index];
         VerifyResult::changed_if(field_changed_at > revision)
@@ -82,11 +82,7 @@ where
         C::TRACKED_FIELD_NAMES[self.field_index]
     }
 
-    fn memo_table_types(&self) -> &Arc<MemoTableTypes> {
+    fn memo_table_types(&self) -> Arc<MemoTableTypes> {
-        unreachable!("tracked field does not allocate pages")
-    }
-
-    fn memo_table_types_mut(&mut self) -> &mut Arc<MemoTableTypes> {
         unreachable!("tracked field does not allocate pages")
     }
 }

src/views.rs

@@ -1,15 +1,10 @@
-use std::{
+use std::any::{Any, TypeId};
-    any::{Any, TypeId},
-    marker::PhantomData,
-    mem,
-    ptr::NonNull,
-};
 
-use crate::{database::RawDatabase, Database};
+use crate::Database;
 
 /// A `Views` struct is associated with some specific database type
 /// (a `DatabaseImpl<U>` for some existential `U`). It contains functions
-/// to downcast to `dyn DbView` for various traits `DbView` via this specific
+/// to downcast from `dyn Database` to `dyn DbView` for various traits `DbView` via this specific
 /// database type.
 /// None of these types are known at compilation time, they are all checked
 /// dynamically through `TypeId` magic.
@@ -18,7 +13,6 @@ pub struct Views {
     view_casters: boxcar::Vec<ViewCaster>,
 }
 
-#[derive(Copy, Clone)]
 struct ViewCaster {
     /// The id of the target type `dyn DbView` that we can cast to.
     target_type_id: TypeId,
@@ -26,69 +20,50 @@ struct ViewCaster {
     /// The name of the target type `dyn DbView` that we can cast to.
     type_name: &'static str,
 
-    /// Type-erased function pointer that downcasts to `dyn DbView`.
+    /// Type-erased function pointer that downcasts from `dyn Database` to `dyn DbView`.
     cast: ErasedDatabaseDownCasterSig,
 }
 
 impl ViewCaster {
-    fn new<DbView: ?Sized + Any>(func: DatabaseDownCasterSig<DbView>) -> ViewCaster {
+    fn new<DbView: ?Sized + Any>(func: unsafe fn(&dyn Database) -> &DbView) -> ViewCaster {
         ViewCaster {
             target_type_id: TypeId::of::<DbView>(),
             type_name: std::any::type_name::<DbView>(),
             // SAFETY: We are type erasing for storage, taking care of unerasing before we call
             // the function pointer.
             cast: unsafe {
-                mem::transmute::<DatabaseDownCasterSig<DbView>, ErasedDatabaseDownCasterSig>(func)
+                std::mem::transmute::<DatabaseDownCasterSig<DbView>, ErasedDatabaseDownCasterSig>(
+                    func,
+                )
             },
         }
     }
 }
 
-type ErasedDatabaseDownCasterSig = unsafe fn(RawDatabase<'_>) -> NonNull<()>;
+type ErasedDatabaseDownCasterSig = unsafe fn(&dyn Database) -> *const ();
-type DatabaseDownCasterSig<DbView> = unsafe fn(RawDatabase<'_>) -> NonNull<DbView>;
+type DatabaseDownCasterSig<DbView> = unsafe fn(&dyn Database) -> &DbView;
 
-#[repr(transparent)]
+pub struct DatabaseDownCaster<DbView: ?Sized>(TypeId, DatabaseDownCasterSig<DbView>);
-pub struct DatabaseDownCaster<DbView: ?Sized>(ViewCaster, PhantomData<fn() -> DbView>);
-
-impl<DbView: ?Sized> Copy for DatabaseDownCaster<DbView> {}
-impl<DbView: ?Sized> Clone for DatabaseDownCaster<DbView> {
-    fn clone(&self) -> Self {
-        *self
-    }
-}
 
 impl<DbView: ?Sized + Any> DatabaseDownCaster<DbView> {
-    /// Downcast `db` to `DbView`.
+    pub fn downcast<'db>(&self, db: &'db dyn Database) -> &'db DbView {
-    ///
+        assert_eq!(
-    /// # Safety
+            self.0,
-    ///
+            db.type_id(),
-    /// The caller must ensure that `db` is of the correct type.
+            "Database type does not match the expected type for this `Views` instance"
-    #[inline]
+        );
-    pub unsafe fn downcast_unchecked<'db>(&self, db: RawDatabase<'db>) -> &'db DbView {
+        // SAFETY: We've asserted that the database is correct.
-        // SAFETY: The caller must ensure that `db` is of the correct type.
+        unsafe { (self.1)(db) }
-        // The returned pointer is live for `'db` due to construction of the downcaster functions.
-        unsafe { (self.unerased_downcaster())(db).as_ref() }
-    }
-    /// Downcast `db` to `DbView`.
-    ///
-    /// # Safety
-    ///
-    /// The caller must ensure that `db` is of the correct type.
-    #[inline]
-    pub unsafe fn downcast_mut_unchecked<'db>(&self, db: RawDatabase<'db>) -> &'db mut DbView {
-        // SAFETY: The caller must ensure that `db` is of the correct type.
-        // The returned pointer is live for `'db` due to construction of the downcaster functions.
-        unsafe { (self.unerased_downcaster())(db).as_mut() }
     }
 
-    #[inline]
+    /// Downcast `db` to `DbView`.
-    fn unerased_downcaster(&self) -> DatabaseDownCasterSig<DbView> {
+    ///
-        // SAFETY: The type-erased function pointer is guaranteed to be ABI compatible for `DbView`
+    /// # Safety
-        unsafe {
+    ///
-            mem::transmute::<ErasedDatabaseDownCasterSig, DatabaseDownCasterSig<DbView>>(
+    /// The caller must ensure that `db` is of the correct type.
-                self.0.cast,
+    pub unsafe fn downcast_unchecked<'db>(&self, db: &'db dyn Database) -> &'db DbView {
-            )
+        // SAFETY: The caller must ensure that `db` is of the correct type.
-        }
+        unsafe { (self.1)(db) }
     }
 }
 
@@ -96,56 +71,43 @@ impl Views {
     pub(crate) fn new<Db: Database>() -> Self {
         let source_type_id = TypeId::of::<Db>();
         let view_casters = boxcar::Vec::new();
-        view_casters.push(ViewCaster::new::<dyn Database>(|db| db.ptr.cast::<Db>()));
+        // special case the no-op transformation, that way we skip out on reconstructing the wide pointer
+        view_casters.push(ViewCaster::new::<dyn Database>(|db| db));
         Self {
             source_type_id,
             view_casters,
         }
     }
 
-    /// Add a new downcaster to `dyn DbView`.
+    /// Add a new downcaster from `dyn Database` to `dyn DbView`.
-    pub fn add<Concrete: 'static, DbView: ?Sized + Any>(
+    pub fn add<DbView: ?Sized + Any>(&self, func: DatabaseDownCasterSig<DbView>) {
-        &self,
-        func: fn(NonNull<Concrete>) -> NonNull<DbView>,
-    ) -> &DatabaseDownCaster<DbView> {
-        assert_eq!(self.source_type_id, TypeId::of::<Concrete>());
         let target_type_id = TypeId::of::<DbView>();
-        if let Some((_, caster)) = self
+        if self
             .view_casters
             .iter()
-            .find(|(_, u)| u.target_type_id == target_type_id)
+            .any(|(_, u)| u.target_type_id == target_type_id)
         {
-            // SAFETY: The type-erased function pointer is guaranteed to be valid for `DbView`
+            return;
-            return unsafe { &*(&raw const *caster).cast::<DatabaseDownCaster<DbView>>() };
         }
-
+        self.view_casters.push(ViewCaster::new::<DbView>(func));
-        // SAFETY: We are type erasing the function pointer for storage, and we will unerase it
-        // before we call it.
-        let caster = unsafe {
-            mem::transmute::<fn(NonNull<Concrete>) -> NonNull<DbView>, DatabaseDownCasterSig<DbView>>(
-                func,
-            )
-        };
-        let caster = ViewCaster::new::<DbView>(caster);
-        let idx = self.view_casters.push(caster);
-        // SAFETY: The type-erased function pointer is guaranteed to be valid for `DbView`
-        unsafe { &*(&raw const self.view_casters[idx]).cast::<DatabaseDownCaster<DbView>>() }
     }
 
-    /// Retrieve an downcaster function to `dyn DbView`.
+    /// Retrieve an downcaster function from `dyn Database` to `dyn DbView`.
     ///
     /// # Panics
     ///
-    /// If the underlying type of `db` is not the same as the database type this downcasts was created for.
+    /// If the underlying type of `db` is not the same as the database type this upcasts was created for.
-    pub fn downcaster_for<DbView: ?Sized + Any>(&self) -> &DatabaseDownCaster<DbView> {
+    pub fn downcaster_for<DbView: ?Sized + Any>(&self) -> DatabaseDownCaster<DbView> {
         let view_type_id = TypeId::of::<DbView>();
-        for (_, view) in self.view_casters.iter() {
+        for (_idx, view) in self.view_casters.iter() {
             if view.target_type_id == view_type_id {
                 // SAFETY: We are unerasing the type erased function pointer having made sure the
                 // TypeId matches.
-                return unsafe {
+                return DatabaseDownCaster(self.source_type_id, unsafe {
-                    &*((view as *const ViewCaster).cast::<DatabaseDownCaster<DbView>>())
+                    std::mem::transmute::<ErasedDatabaseDownCasterSig, DatabaseDownCasterSig<DbView>>(
-                };
+                        view.cast,
+                    )
+                });
             }
         }
 

src/zalsa.rs

@@ -1,15 +1,18 @@
 use std::any::{Any, TypeId};
 use std::hash::BuildHasherDefault;
+use std::marker::PhantomData;
+use std::mem;
+use std::num::NonZeroU32;
 use std::panic::RefUnwindSafe;
 
-use hashbrown::HashMap;
 use rustc_hash::FxHashMap;
 
-use crate::database::RawDatabase;
 use crate::hash::TypeIdHasher;
 use crate::ingredient::{Ingredient, Jar};
-use crate::plumbing::SalsaStructInDb;
+use crate::nonce::{Nonce, NonceGenerator};
 use crate::runtime::Runtime;
+use crate::sync::atomic::{AtomicU64, Ordering};
+use crate::sync::{papaya, Mutex, RwLock};
 use crate::table::memo::MemoTableWithTypes;
 use crate::table::Table;
 use crate::views::Views;
@@ -53,20 +56,19 @@ pub unsafe trait ZalsaDatabase: Any {
 
     /// Clone the database.
     #[doc(hidden)]
-    fn fork_db(&self) -> RawDatabase<'static>;
+    fn fork_db(&self) -> Box<dyn Database>;
 }
 
 pub fn views<Db: ?Sized + Database>(db: &Db) -> &Views {
     db.zalsa().views()
 }
+
 /// Nonce type representing the underlying database storage.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
-#[cfg(not(feature = "inventory"))]
 pub struct StorageNonce;
 
 // Generator for storage nonces.
-#[cfg(not(feature = "inventory"))]
+static NONCE: NonceGenerator<StorageNonce> = NonceGenerator::new();
-static NONCE: crate::nonce::NonceGenerator<StorageNonce> = crate::nonce::NonceGenerator::new();
 
 /// An ingredient index identifies a particular [`Ingredient`] in the database.
 ///
@@ -81,20 +83,10 @@ impl IngredientIndex {
     /// This reserves one bit for an optional tag.
     const MAX_INDEX: u32 = 0x7FFF_FFFF;
 
-    /// Create an ingredient index from a `u32`.
+    /// Create an ingredient index from a `usize`.
-    pub(crate) fn new(v: u32) -> Self {
+    pub(crate) fn from(v: usize) -> Self {
-        assert!(v <= Self::MAX_INDEX);
+        assert!(v <= Self::MAX_INDEX as usize);
-        Self(v)
+        Self(v as u32)
-    }
-
-    /// Create an ingredient index from a `u32`, without performing validating
-    /// that the index is valid.
-    ///
-    /// # Safety
-    ///
-    /// The index must be less than or equal to `IngredientIndex::MAX_INDEX`.
-    pub(crate) unsafe fn new_unchecked(v: u32) -> Self {
-        Self(v)
     }
 
     /// Convert the ingredient index back into a `u32`.
@@ -142,24 +134,28 @@ impl MemoIngredientIndex {
 pub struct Zalsa {
     views_of: Views,
 
-    #[cfg(not(feature = "inventory"))]
+    nonce: Nonce<StorageNonce>,
-    nonce: crate::nonce::Nonce<StorageNonce>,
 
     /// Map from the [`IngredientIndex::as_usize`][] of a salsa struct to a list of
     /// [ingredient-indices](`IngredientIndex`) for tracked functions that have this salsa struct
     /// as input.
-    memo_ingredient_indices: Vec<Vec<IngredientIndex>>,
+    memo_ingredient_indices: RwLock<Vec<Vec<IngredientIndex>>>,
 
     /// Map from the type-id of an `impl Jar` to the index of its first ingredient.
-    jar_map: HashMap<TypeId, IngredientIndex, BuildHasherDefault<TypeIdHasher>>,
+    jar_map: papaya::HashMap<TypeId, IngredientIndex, BuildHasherDefault<TypeIdHasher>>,
+
+    /// The write-lock for `jar_map`.
+    jar_map_lock: Mutex<()>,
 
     /// A map from the `IngredientIndex` to the `TypeId` of its ID struct.
     ///
     /// Notably this is not the reverse mapping of `jar_map`.
-    ingredient_to_id_struct_type_id_map: FxHashMap<IngredientIndex, TypeId>,
+    ingredient_to_id_struct_type_id_map: RwLock<FxHashMap<IngredientIndex, TypeId>>,
 
     /// Vector of ingredients.
-    ingredients_vec: Vec<Box<dyn Ingredient>>,
+    ///
+    /// Immutable unless the mutex on `ingredients_map` is held.
+    ingredients_vec: boxcar::Vec<Box<dyn Ingredient>>,
 
     /// Indices of ingredients that require reset when a new revision starts.
     ingredients_requiring_reset: boxcar::Vec<IngredientIndex>,
@@ -181,43 +177,22 @@ impl RefUnwindSafe for Zalsa {}
 impl Zalsa {
     pub(crate) fn new<Db: Database>(
         event_callback: Option<Box<dyn Fn(crate::Event) + Send + Sync + 'static>>,
-        jars: Vec<ErasedJar>,
     ) -> Self {
-        let mut zalsa = Self {
+        Self {
             views_of: Views::new::<Db>(),
-            jar_map: HashMap::default(),
+            nonce: NONCE.nonce(),
+            jar_map: papaya::HashMap::default(),
+            jar_map_lock: Mutex::default(),
             ingredient_to_id_struct_type_id_map: Default::default(),
-            ingredients_vec: Vec::new(),
+            ingredients_vec: boxcar::Vec::new(),
             ingredients_requiring_reset: boxcar::Vec::new(),
             runtime: Runtime::default(),
             memo_ingredient_indices: Default::default(),
             event_callback,
-            #[cfg(not(feature = "inventory"))]
-            nonce: NONCE.nonce(),
-        };
-
-        // Collect and initialize all registered ingredients.
-        #[cfg(feature = "inventory")]
-        let mut jars = inventory::iter::<ErasedJar>()
-            .copied()
-            .chain(jars)
-            .collect::<Vec<_>>();
-
-        #[cfg(not(feature = "inventory"))]
-        let mut jars = jars;
-
-        // Ensure structs are initialized before tracked functions.
-        jars.sort_by_key(|jar| jar.kind);
-
-        for jar in jars {
-            zalsa.insert_jar(jar);
         }
-
-        zalsa
     }
 
-    #[cfg(not(feature = "inventory"))]
+    pub(crate) fn nonce(&self) -> Nonce<StorageNonce> {
-    pub(crate) fn nonce(&self) -> crate::nonce::Nonce<StorageNonce> {
         self.nonce
     }
 
@@ -231,19 +206,19 @@ impl Zalsa {
 
     /// Returns the [`Table`] used to store the value of salsa structs
     #[inline]
-    pub fn table(&self) -> &Table {
+    pub(crate) fn table(&self) -> &Table {
         self.runtime.table()
     }
 
     /// Returns the [`MemoTable`][] for the salsa struct with the given id
-    pub(crate) fn memo_table_for<T: SalsaStructInDb>(&self, id: Id) -> MemoTableWithTypes<'_> {
+    pub(crate) fn memo_table_for(&self, id: Id) -> MemoTableWithTypes<'_> {
-        // SAFETY: We are supplying the correct current revision.
+        let table = self.table();
-        unsafe { T::memo_table(self, id, self.current_revision()) }
+        // SAFETY: We are supplying the correct current revision
+        unsafe { table.memos(id, self.current_revision()) }
     }
 
-    /// Returns the ingredient at the given index, or panics if it is out-of-bounds.
     #[inline]
-    pub fn lookup_ingredient(&self, index: IngredientIndex) -> &dyn Ingredient {
+    pub(crate) fn lookup_ingredient(&self, index: IngredientIndex) -> &dyn Ingredient {
         let index = index.as_u32() as usize;
         self.ingredients_vec
             .get(index)
@@ -251,25 +226,12 @@ impl Zalsa {
             .as_ref()
     }
 
-    /// Returns the ingredient at the given index.
-    ///
-    /// # Safety
-    ///
-    /// The index must be in-bounds.
-    #[inline]
-    pub unsafe fn lookup_ingredient_unchecked(&self, index: IngredientIndex) -> &dyn Ingredient {
-        let index = index.as_u32() as usize;
-
-        // SAFETY: Guaranteed by caller.
-        unsafe { self.ingredients_vec.get_unchecked(index).as_ref() }
-    }
-
     pub(crate) fn ingredient_index_for_memo(
         &self,
         struct_ingredient_index: IngredientIndex,
         memo_ingredient_index: MemoIngredientIndex,
     ) -> IngredientIndex {
-        self.memo_ingredient_indices[struct_ingredient_index.as_u32() as usize]
+        self.memo_ingredient_indices.read()[struct_ingredient_index.as_u32() as usize]
             [memo_ingredient_index.as_usize()]
     }
 
@@ -277,7 +239,7 @@ impl Zalsa {
     pub(crate) fn ingredients(&self) -> impl Iterator<Item = &dyn Ingredient> {
         self.ingredients_vec
             .iter()
-            .map(|ingredient| ingredient.as_ref())
+            .map(|(_, ingredient)| ingredient.as_ref())
     }
 
     /// Starts unwinding the stack if the current revision is cancelled.
@@ -297,11 +259,11 @@ impl Zalsa {
     }
 
     pub(crate) fn next_memo_ingredient_index(
-        &mut self,
+        &self,
         struct_ingredient_index: IngredientIndex,
         ingredient_index: IngredientIndex,
     ) -> MemoIngredientIndex {
-        let memo_ingredients = &mut self.memo_ingredient_indices;
+        let mut memo_ingredients = self.memo_ingredient_indices.write();
         let idx = struct_ingredient_index.as_u32() as usize;
         let memo_ingredients = if let Some(memo_ingredients) = memo_ingredients.get_mut(idx) {
             memo_ingredients
@@ -329,6 +291,7 @@ impl Zalsa {
         let ingredient_index = self.ingredient_index(id);
         *self
             .ingredient_to_id_struct_type_id_map
+            .read()
             .get(&ingredient_index)
             .expect("should have the ingredient index available")
     }
@@ -336,36 +299,44 @@ impl Zalsa {
     /// **NOT SEMVER STABLE**
     #[doc(hidden)]
     #[inline]
-    pub fn lookup_jar_by_type<J: Jar>(&self) -> IngredientIndex {
+    pub fn lookup_jar_by_type<J: Jar>(&self) -> JarEntry<'_, J> {
         let jar_type_id = TypeId::of::<J>();
+        let guard = self.jar_map.guard();
 
-        *self.jar_map.get(&jar_type_id).unwrap_or_else(|| {
+        match self.jar_map.get(&jar_type_id, &guard) {
-            panic!(
+            Some(index) => JarEntry::Occupied(index),
-                "ingredient `{}` was not registered",
+            None => JarEntry::Vacant {
-                std::any::type_name::<J>()
+                guard,
-            )
+                zalsa: self,
-        })
+                _jar: PhantomData,
+            },
+        }
     }
 
-    fn insert_jar(&mut self, jar: ErasedJar) {
+    #[cold]
-        let jar_type_id = (jar.type_id)();
+    #[inline(never)]
+    fn add_or_lookup_jar_by_type<J: Jar>(&self, guard: &papaya::LocalGuard<'_>) -> IngredientIndex {
+        let jar_type_id = TypeId::of::<J>();
+        let dependencies = J::create_dependencies(self);
 
-        let index = IngredientIndex::new(self.ingredients_vec.len() as u32);
+        let jar_map_lock = self.jar_map_lock.lock();
 
-        if self.jar_map.contains_key(&jar_type_id) {
+        let index = IngredientIndex::from(self.ingredients_vec.count());
-            return;
-        }
 
-        let ingredients = (jar.create_ingredients)(self, index);
+        // Someone made it earlier than us.
+        if let Some(index) = self.jar_map.get(&jar_type_id, guard) {
+            return index;
+        };
+
+        let ingredients = J::create_ingredients(self, index, dependencies);
         for ingredient in ingredients {
             let expected_index = ingredient.ingredient_index();
+
             if ingredient.requires_reset_for_new_revision() {
                 self.ingredients_requiring_reset.push(expected_index);
             }
 
-            self.ingredients_vec.push(ingredient);
+            let actual_index = self.ingredients_vec.push(ingredient);
-
-            let actual_index = self.ingredients_vec.len() - 1;
             assert_eq!(
                 expected_index.as_u32() as usize,
                 actual_index,
@@ -376,10 +347,17 @@ impl Zalsa {
             );
         }
 
-        self.jar_map.insert(jar_type_id, index);
+        // Insert the index after all ingredients are inserted to avoid exposing
+        // partially initialized jars to readers.
+        self.jar_map.insert(jar_type_id, index, guard);
+
+        drop(jar_map_lock);
 
         self.ingredient_to_id_struct_type_id_map
-            .insert(index, (jar.id_struct_type_id)());
+            .write()
+            .insert(index, J::id_struct_type_id());
+
+        index
     }
 
     /// **NOT SEMVER STABLE**
@@ -415,7 +393,7 @@ impl Zalsa {
     #[doc(hidden)]
     pub fn new_revision(&mut self) -> Revision {
         let new_revision = self.runtime.new_revision();
-        let _span = crate::tracing::debug_span!("new_revision", ?new_revision).entered();
+        let _span = tracing::debug_span!("new_revision", ?new_revision).entered();
 
         for (_, index) in self.ingredients_requiring_reset.iter() {
             let index = index.as_u32() as usize;
@@ -433,7 +411,7 @@ impl Zalsa {
     /// **NOT SEMVER STABLE**
     #[doc(hidden)]
     pub fn evict_lru(&mut self) {
-        let _span = crate::tracing::debug_span!("evict_lru").entered();
+        let _span = tracing::debug_span!("evict_lru").entered();
         for (_, index) in self.ingredients_requiring_reset.iter() {
             let index = index.as_u32() as usize;
             self.ingredients_vec
@@ -450,83 +428,145 @@ impl Zalsa {
 
     #[inline(always)]
     pub fn event(&self, event: &dyn Fn() -> crate::Event) {
-        if self.event_callback.is_some() {
+        if let Some(event_callback) = &self.event_callback {
-            self.event_cold(event);
+            event_callback(event());
+        }
+    }
+}
+
+pub enum JarEntry<'a, J> {
+    Occupied(IngredientIndex),
+    Vacant {
+        zalsa: &'a Zalsa,
+        guard: papaya::LocalGuard<'a>,
+        _jar: PhantomData<J>,
+    },
+}
+
+impl<J> JarEntry<'_, J>
+where
+    J: Jar,
+{
+    #[inline]
+    pub fn get(&self) -> Option<IngredientIndex> {
+        match *self {
+            JarEntry::Occupied(index) => Some(index),
+            JarEntry::Vacant { .. } => None,
         }
     }
 
-    // Avoid inlining, as events are typically only enabled for debugging purposes.
+    #[inline]
-    #[cold]
+    pub fn get_or_create(&self) -> IngredientIndex {
-    #[inline(never)]
+        match self {
-    pub fn event_cold(&self, event: &dyn Fn() -> crate::Event) {
+            JarEntry::Occupied(index) => *index,
-        let event_callback = self.event_callback.as_ref().unwrap();
+            JarEntry::Vacant { zalsa, guard, _jar } => zalsa.add_or_lookup_jar_by_type::<J>(guard),
-        event_callback(event());
+        }
     }
 }
 
-/// A type-erased `Jar`, used for ingredient registration.
+/// Caches a pointer to an ingredient in a database.
-#[derive(Clone, Copy)]
+/// Optimized for the case of a single database.
-pub struct ErasedJar {
+pub struct IngredientCache<I>
-    kind: JarKind,
+where
-    type_id: fn() -> TypeId,
+    I: Ingredient,
-    id_struct_type_id: fn() -> TypeId,
+{
-    create_ingredients: fn(&mut Zalsa, IngredientIndex) -> Vec<Box<dyn Ingredient>>,
+    // A packed representation of `Option<(Nonce<StorageNonce>, IngredientIndex)>`.
+    //
+    // This allows us to replace a lock in favor of an atomic load. This works thanks to `Nonce`
+    // having a niche, which means the entire type can fit into an `AtomicU64`.
+    cached_data: AtomicU64,
+    phantom: PhantomData<fn() -> I>,
 }
 
-/// The kind of an `Jar`.
+impl<I> Default for IngredientCache<I>
-///
+where
-/// Note that the ordering of the variants is important. Struct ingredients must be
+    I: Ingredient,
-/// initialized before tracked functions, as tracked function ingredients depend on
+{
-/// their input struct.
+    fn default() -> Self {
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug)]
+        Self::new()
-pub enum JarKind {
+    }
-    /// An input/tracked/interned struct.
-    Struct,
-
-    /// A tracked function.
-    TrackedFn,
 }
 
-impl ErasedJar {
+impl<I> IngredientCache<I>
-    /// Performs type-erasure of a given ingredient.
+where
-    pub const fn erase<I: HasJar>() -> Self {
+    I: Ingredient,
+{
+    const UNINITIALIZED: u64 = 0;
+
+    /// Create a new cache
+    pub const fn new() -> Self {
         Self {
-            kind: I::KIND,
+            cached_data: AtomicU64::new(Self::UNINITIALIZED),
-            type_id: TypeId::of::<I::Jar>,
+            phantom: PhantomData,
-            create_ingredients: <I::Jar>::create_ingredients,
-            id_struct_type_id: <I::Jar>::id_struct_type_id,
         }
     }
+
+    /// Get a reference to the ingredient in the database.
+    /// If the ingredient is not already in the cache, it will be created.
+    #[inline(always)]
+    pub fn get_or_create<'db>(
+        &self,
+        zalsa: &'db Zalsa,
+        create_index: impl Fn() -> IngredientIndex,
+    ) -> &'db I {
+        let index = self.get_or_create_index(zalsa, create_index);
+        zalsa.lookup_ingredient(index).assert_type::<I>()
+    }
+
+    /// Get a reference to the ingredient in the database.
+    /// If the ingredient is not already in the cache, it will be created.
+    #[inline(always)]
+    pub fn get_or_create_index(
+        &self,
+        zalsa: &Zalsa,
+        create_index: impl Fn() -> IngredientIndex,
+    ) -> IngredientIndex {
+        const _: () = assert!(
+            mem::size_of::<(Nonce<StorageNonce>, IngredientIndex)>() == mem::size_of::<u64>()
+        );
+        let cached_data = self.cached_data.load(Ordering::Acquire);
+        if cached_data == Self::UNINITIALIZED {
+            #[cold]
+            #[inline(never)]
+            fn get_or_create_index_slow<I: Ingredient>(
+                this: &IngredientCache<I>,
+                zalsa: &Zalsa,
+                create_index: impl Fn() -> IngredientIndex,
+            ) -> IngredientIndex {
+                let index = create_index();
+                let nonce = zalsa.nonce().into_u32().get() as u64;
+                let packed = (nonce << u32::BITS) | (index.as_u32() as u64);
+                debug_assert_ne!(packed, IngredientCache::<I>::UNINITIALIZED);
+
+                // Discard the result, whether we won over the cache or not does not matter
+                // we know that something has been cached now
+                _ = this.cached_data.compare_exchange(
+                    IngredientCache::<I>::UNINITIALIZED,
+                    packed,
+                    Ordering::Release,
+                    Ordering::Acquire,
+                );
+                // and we already have our index computed so we can just use that
+                index
+            }
+
+            return get_or_create_index_slow(self, zalsa, create_index);
+        };
+
+        // unpack our u64
+        // SAFETY: We've checked against `UNINITIALIZED` (0) above and so the upper bits must be non-zero
+        let nonce = Nonce::<StorageNonce>::from_u32(unsafe {
+            NonZeroU32::new_unchecked((cached_data >> u32::BITS) as u32)
+        });
+        let mut index = IngredientIndex(cached_data as u32);
+
+        if zalsa.nonce() != nonce {
+            index = create_index();
+        }
+        index
+    }
 }
 
-/// A salsa ingredient that can be registered in the database.
-///
-/// This trait is implemented for tracked functions and salsa structs.
-pub trait HasJar {
-    /// The [`Jar`] associated with this ingredient.
-    type Jar: Jar;
-
-    /// The [`JarKind`] for `Self::Jar`.
-    const KIND: JarKind;
-}
-
-// Collect jars statically at compile-time if supported.
-#[cfg(feature = "inventory")]
-inventory::collect!(ErasedJar);
-
-#[cfg(feature = "inventory")]
-pub use inventory::submit as register_jar;
-
-#[cfg(not(feature = "inventory"))]
-#[macro_export]
-#[doc(hidden)]
-macro_rules! register_jar {
-    ($($_:tt)*) => {};
-}
-
-#[cfg(not(feature = "inventory"))]
-pub use crate::register_jar;
-
 /// Given a wide pointer `T`, extracts the data pointer (typed as `U`).
 ///
 /// # Safety

src/zalsa_local.rs

@@ -1,15 +1,12 @@
-use std::cell::{RefCell, UnsafeCell};
+use std::cell::RefCell;
 use std::panic::UnwindSafe;
 use std::ptr::{self, NonNull};
 
 use rustc_hash::FxHashMap;
 use thin_vec::ThinVec;
+use tracing::debug;
 
-#[cfg(feature = "accumulator")]
+use crate::accumulator::accumulated_map::{AccumulatedMap, AtomicInputAccumulatedValues};
-use crate::accumulator::{
-    accumulated_map::{AccumulatedMap, AtomicInputAccumulatedValues},
-    Accumulator,
-};
 use crate::active_query::QueryStack;
 use crate::cycle::{empty_cycle_heads, CycleHeads, IterationCount};
 use crate::durability::Durability;
@@ -19,7 +16,7 @@ use crate::sync::atomic::AtomicBool;
 use crate::table::{PageIndex, Slot, Table};
 use crate::tracked_struct::{Disambiguator, Identity, IdentityHash, IdentityMap};
 use crate::zalsa::{IngredientIndex, Zalsa};
-use crate::{Cancelled, Id, Revision};
+use crate::{Accumulator, Cancelled, Id, Revision};
 
 /// State that is specific to a single execution thread.
 ///
@@ -36,14 +33,14 @@ pub struct ZalsaLocal {
 
     /// Stores the most recent page for a given ingredient.
     /// This is thread-local to avoid contention.
-    most_recent_pages: UnsafeCell<FxHashMap<IngredientIndex, PageIndex>>,
+    most_recent_pages: RefCell<FxHashMap<IngredientIndex, PageIndex>>,
 }
 
 impl ZalsaLocal {
     pub(crate) fn new() -> Self {
         ZalsaLocal {
             query_stack: RefCell::new(QueryStack::default()),
-            most_recent_pages: UnsafeCell::new(FxHashMap::default()),
+            most_recent_pages: RefCell::new(FxHashMap::default()),
         }
     }
 
@@ -57,60 +54,35 @@ impl ZalsaLocal {
     /// Allocate a new id in `table` for the given ingredient
     /// storing `value`. Remembers the most recent page from this
     /// thread and attempts to reuse it.
-    pub(crate) fn allocate<'db, T: Slot>(
+    pub(crate) fn allocate<T: Slot>(
         &self,
-        zalsa: &'db Zalsa,
+        zalsa: &Zalsa,
         ingredient: IngredientIndex,
         mut value: impl FnOnce(Id) -> T,
-    ) -> (Id, &'db T) {
+    ) -> Id {
-        // SAFETY: `ZalsaLocal` is `!Sync`, and we never expose a reference to this field,
-        // so we have exclusive access.
-        let most_recent_pages = unsafe { &mut *self.most_recent_pages.get() };
-
-        // Fast-path, we already have an unfilled page available.
-        if let Some(&page) = most_recent_pages.get(&ingredient) {
-            let page_ref = zalsa.table().page::<T>(page);
-            match page_ref.allocate(page, value) {
-                Ok((id, value)) => return (id, value),
-                Err(v) => value = v,
-            }
-        }
-
-        self.allocate_cold(zalsa, ingredient, value)
-    }
-
-    #[cold]
-    #[inline(never)]
-    pub(crate) fn allocate_cold<'db, T: Slot>(
-        &self,
-        zalsa: &'db Zalsa,
-        ingredient: IngredientIndex,
-        mut value: impl FnOnce(Id) -> T,
-    ) -> (Id, &'db T) {
         let memo_types = || {
             zalsa
                 .lookup_ingredient(ingredient)
                 .memo_table_types()
                 .clone()
         };
-
-        // SAFETY: `ZalsaLocal` is `!Sync`, and we never expose a reference to this field,
-        // so we have exclusive access.
-        let most_recent_pages = unsafe { &mut *self.most_recent_pages.get() };
-
         // Find the most recent page, pushing a page if needed
-        let mut page = *most_recent_pages.entry(ingredient).or_insert_with(|| {
+        let mut page = *self
-            zalsa
+            .most_recent_pages
-                .table()
+            .borrow_mut()
-                .fetch_or_push_page::<T>(ingredient, memo_types)
+            .entry(ingredient)
-        });
+            .or_insert_with(|| {
+                zalsa
+                    .table()
+                    .fetch_or_push_page::<T>(ingredient, memo_types)
+            });
 
         loop {
             // Try to allocate an entry on that page
             let page_ref = zalsa.table().page::<T>(page);
             match page_ref.allocate(page, value) {
                 // If successful, return
-                Ok((id, value)) => return (id, value),
+                Ok(id) => return id,
 
                 // Otherwise, create a new page and try again
                 // Note that we could try fetching a page again, but as we just filled one up
@@ -118,7 +90,7 @@ impl ZalsaLocal {
                 Err(v) => {
                     value = v;
                     page = zalsa.table().push_page::<T>(ingredient, memo_types());
-                    most_recent_pages.insert(ingredient, page);
+                    self.most_recent_pages.borrow_mut().insert(ingredient, page);
                 }
             }
         }
@@ -130,124 +102,88 @@ impl ZalsaLocal {
         database_key_index: DatabaseKeyIndex,
         iteration_count: IterationCount,
     ) -> ActiveQueryGuard<'_> {
-        // SAFETY: We do not access the query stack reentrantly.
+        let mut query_stack = self.query_stack.borrow_mut();
-        unsafe {
+        query_stack.push_new_query(database_key_index, iteration_count);
-            self.with_query_stack_unchecked_mut(|stack| {
+        ActiveQueryGuard {
-                stack.push_new_query(database_key_index, iteration_count);
+            local_state: self,
-
+            database_key_index,
-                ActiveQueryGuard {
+            #[cfg(debug_assertions)]
-                    local_state: self,
+            push_len: query_stack.len(),
-                    database_key_index,
-                    #[cfg(debug_assertions)]
-                    push_len: stack.len(),
-                }
-            })
         }
     }
 
     /// Executes a closure within the context of the current active query stacks (mutable).
-    ///
-    /// # Safety
-    ///
-    /// The closure cannot access the query stack reentrantly, whether mutable or immutable.
     #[inline(always)]
-    pub(crate) unsafe fn with_query_stack_unchecked_mut<R>(
+    pub(crate) fn with_query_stack_mut<R>(
         &self,
-        f: impl UnwindSafe + FnOnce(&mut QueryStack) -> R,
+        c: impl UnwindSafe + FnOnce(&mut QueryStack) -> R,
     ) -> R {
-        // SAFETY: The caller guarantees that the query stack will not be accessed reentrantly.
+        c(&mut self.query_stack.borrow_mut())
-        // Additionally, `ZalsaLocal` is `!Sync`, and we never expose a reference to the query
-        // stack except through this method, so we have exclusive access.
-        unsafe { f(&mut self.query_stack.try_borrow_mut().unwrap_unchecked()) }
     }
 
-    /// Executes a closure within the context of the current active query stacks.
-    ///
-    /// # Safety
-    ///
-    /// No mutable references to the query stack can exist while the closure is executed.
     #[inline(always)]
-    pub(crate) unsafe fn with_query_stack_unchecked<R>(
+    pub(crate) fn with_query_stack<R>(&self, c: impl UnwindSafe + FnOnce(&QueryStack) -> R) -> R {
-        &self,
+        c(&mut self.query_stack.borrow())
-        f: impl UnwindSafe + FnOnce(&QueryStack) -> R,
-    ) -> R {
-        // SAFETY: The caller guarantees that the query stack will not being accessed mutably.
-        // Additionally, `ZalsaLocal` is `!Sync`, and we never expose a reference to the query
-        // stack except through this method, so we have exclusive access.
-        unsafe { f(&self.query_stack.try_borrow().unwrap_unchecked()) }
     }
 
     #[inline(always)]
     pub(crate) fn try_with_query_stack<R>(
         &self,
-        f: impl UnwindSafe + FnOnce(&QueryStack) -> R,
+        c: impl UnwindSafe + FnOnce(&QueryStack) -> R,
     ) -> Option<R> {
         self.query_stack
             .try_borrow()
             .ok()
             .as_ref()
-            .map(|stack| f(stack))
+            .map(|stack| c(stack))
     }
 
     /// Returns the index of the active query along with its *current* durability/changed-at
     /// information. As the query continues to execute, naturally, that information may change.
     pub(crate) fn active_query(&self) -> Option<(DatabaseKeyIndex, Stamp)> {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack(|stack| {
-        unsafe {
+            stack
-            self.with_query_stack_unchecked(|stack| {
+                .last()
-                stack
+                .map(|active_query| (active_query.database_key_index, active_query.stamp()))
-                    .last()
+        })
-                    .map(|active_query| (active_query.database_key_index, active_query.stamp()))
-            })
-        }
     }
 
     /// Add an output to the current query's list of dependencies
     ///
     /// Returns `Err` if not in a query.
-    #[cfg(feature = "accumulator")]
     pub(crate) fn accumulate<A: Accumulator>(
         &self,
         index: IngredientIndex,
         value: A,
     ) -> Result<(), ()> {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            if let Some(top_query) = stack.last_mut() {
-            self.with_query_stack_unchecked_mut(|stack| {
+                top_query.accumulate(index, value);
-                if let Some(top_query) = stack.last_mut() {
+                Ok(())
-                    top_query.accumulate(index, value);
+            } else {
-                    Ok(())
+                Err(())
-                } else {
+            }
-                    Err(())
+        })
-                }
-            })
-        }
     }
 
     /// Add an output to the current query's list of dependencies
     pub(crate) fn add_output(&self, entity: DatabaseKeyIndex) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            if let Some(top_query) = stack.last_mut() {
-            self.with_query_stack_unchecked_mut(|stack| {
+                top_query.add_output(entity)
-                if let Some(top_query) = stack.last_mut() {
+            }
-                    top_query.add_output(entity)
+        })
-                }
-            })
-        }
     }
 
     /// Check whether `entity` is an output of the currently active query (if any)
     pub(crate) fn is_output_of_active_query(&self, entity: DatabaseKeyIndex) -> bool {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            if let Some(top_query) = stack.last_mut() {
-            self.with_query_stack_unchecked_mut(|stack| {
+                top_query.is_output(entity)
-                if let Some(top_query) = stack.last_mut() {
+            } else {
-                    top_query.is_output(entity)
+                false
-                } else {
+            }
-                    false
+        })
-                }
-            })
-        }
     }
 
     /// Register that currently active query reads the given input
@@ -257,34 +193,26 @@ impl ZalsaLocal {
         input: DatabaseKeyIndex,
         durability: Durability,
         changed_at: Revision,
+        has_accumulated: bool,
+        accumulated_inputs: &AtomicInputAccumulatedValues,
         cycle_heads: &CycleHeads,
-        #[cfg(feature = "accumulator")] has_accumulated: bool,
-        #[cfg(feature = "accumulator")] accumulated_inputs: &AtomicInputAccumulatedValues,
     ) {
-        crate::tracing::debug!(
+        debug!(
             "report_tracked_read(input={:?}, durability={:?}, changed_at={:?})",
-            input,
+            input, durability, changed_at
-            durability,
-            changed_at
         );
-
+        self.with_query_stack_mut(|stack| {
-        // SAFETY: We do not access the query stack reentrantly.
+            if let Some(top_query) = stack.last_mut() {
-        unsafe {
+                top_query.add_read(
-            self.with_query_stack_unchecked_mut(|stack| {
+                    input,
-                if let Some(top_query) = stack.last_mut() {
+                    durability,
-                    top_query.add_read(
+                    changed_at,
-                        input,
+                    has_accumulated,
-                        durability,
+                    accumulated_inputs,
-                        changed_at,
+                    cycle_heads,
-                        cycle_heads,
+                );
-                        #[cfg(feature = "accumulator")]
+            }
-                        has_accumulated,
+        })
-                        #[cfg(feature = "accumulator")]
-                        accumulated_inputs,
-                    );
-                }
-            })
-        }
     }
 
     /// Register that currently active query reads the given input
@@ -295,21 +223,15 @@ impl ZalsaLocal {
         durability: Durability,
         changed_at: Revision,
     ) {
-        crate::tracing::debug!(
+        debug!(
             "report_tracked_read(input={:?}, durability={:?}, changed_at={:?})",
-            input,
+            input, durability, changed_at
-            durability,
-            changed_at
         );
-
+        self.with_query_stack_mut(|stack| {
-        // SAFETY: We do not access the query stack reentrantly.
+            if let Some(top_query) = stack.last_mut() {
-        unsafe {
+                top_query.add_read_simple(input, durability, changed_at);
-            self.with_query_stack_unchecked_mut(|stack| {
+            }
-                if let Some(top_query) = stack.last_mut() {
+        })
-                    top_query.add_read_simple(input, durability, changed_at);
-                }
-            })
-        }
     }
 
     /// Register that the current query read an untracked value
@@ -319,14 +241,11 @@ impl ZalsaLocal {
     /// * `current_revision`, the current revision
     #[inline(always)]
     pub(crate) fn report_untracked_read(&self, current_revision: Revision) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            if let Some(top_query) = stack.last_mut() {
-            self.with_query_stack_unchecked_mut(|stack| {
+                top_query.add_untracked_read(current_revision);
-                if let Some(top_query) = stack.last_mut() {
+            }
-                    top_query.add_untracked_read(current_revision);
+        })
-                }
-            })
-        }
     }
 
     /// Update the top query on the stack to act as though it read a value
@@ -334,14 +253,11 @@ impl ZalsaLocal {
     // FIXME: Use or remove this.
     #[allow(dead_code)]
     pub(crate) fn report_synthetic_read(&self, durability: Durability, revision: Revision) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            if let Some(top_query) = stack.last_mut() {
-            self.with_query_stack_unchecked_mut(|stack| {
+                top_query.add_synthetic_read(durability, revision);
-                if let Some(top_query) = stack.last_mut() {
+            }
-                    top_query.add_synthetic_read(durability, revision);
+        })
-                }
-            })
-        }
     }
 
     /// Called when the active queries creates an index from the
@@ -356,42 +272,33 @@ impl ZalsaLocal {
     ///   * the disambiguator index
     #[track_caller]
     pub(crate) fn disambiguate(&self, key: IdentityHash) -> (Stamp, Disambiguator) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            let top_query = stack.last_mut().expect(
-            self.with_query_stack_unchecked_mut(|stack| {
+                "cannot create a tracked struct disambiguator outside of a tracked function",
-                let top_query = stack.last_mut().expect(
+            );
-                    "cannot create a tracked struct disambiguator outside of a tracked function",
+            let disambiguator = top_query.disambiguate(key);
-                );
+            (top_query.stamp(), disambiguator)
-                let disambiguator = top_query.disambiguate(key);
+        })
-                (top_query.stamp(), disambiguator)
-            })
-        }
     }
 
     #[track_caller]
     pub(crate) fn tracked_struct_id(&self, identity: &Identity) -> Option<Id> {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack(|stack| {
-        unsafe {
+            let top_query = stack
-            self.with_query_stack_unchecked(|stack| {
+                .last()
-                let top_query = stack
+                .expect("cannot create a tracked struct ID outside of a tracked function");
-                    .last()
+            top_query.tracked_struct_ids().get(identity)
-                    .expect("cannot create a tracked struct ID outside of a tracked function");
+        })
-                top_query.tracked_struct_ids().get(identity)
-            })
-        }
     }
 
     #[track_caller]
     pub(crate) fn store_tracked_struct_id(&self, identity: Identity, id: Id) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.with_query_stack_mut(|stack| {
-        unsafe {
+            let top_query = stack
-            self.with_query_stack_unchecked_mut(|stack| {
+                .last_mut()
-                let top_query = stack
+                .expect("cannot store a tracked struct ID outside of a tracked function");
-                    .last_mut()
+            top_query.tracked_struct_ids_mut().insert(identity, id);
-                    .expect("cannot store a tracked struct ID outside of a tracked function");
+        })
-                top_query.tracked_struct_ids_mut().insert(identity, id);
-            })
-        }
     }
 
     #[cold]
@@ -427,7 +334,6 @@ pub(crate) struct QueryRevisions {
     ///
     /// Note that this field could be in `QueryRevisionsExtra` as it is only relevant
     /// for accumulators, but we get it for free anyways due to padding.
-    #[cfg(feature = "accumulator")]
     pub(super) accumulated_inputs: AtomicInputAccumulatedValues,
 
     /// Are the `cycle_heads` verified to not be provisional anymore?
@@ -447,11 +353,10 @@ impl QueryRevisions {
         let QueryRevisions {
             changed_at: _,
             durability: _,
+            accumulated_inputs: _,
             verified_final: _,
             origin,
             extra,
-            #[cfg(feature = "accumulator")]
-                accumulated_inputs: _,
         } = self;
 
         let mut memory = 0;
@@ -481,24 +386,19 @@ pub(crate) struct QueryRevisionsExtra(Option<Box<QueryRevisionsExtraInner>>);
 
 impl QueryRevisionsExtra {
     pub fn new(
-        #[cfg(feature = "accumulator")] accumulated: AccumulatedMap,
+        accumulated: AccumulatedMap,
         tracked_struct_ids: IdentityMap,
         cycle_heads: CycleHeads,
         iteration: IterationCount,
     ) -> Self {
-        #[cfg(feature = "accumulator")]
+        let inner = if tracked_struct_ids.is_empty()
-        let acc = accumulated.is_empty();
-        #[cfg(not(feature = "accumulator"))]
-        let acc = true;
-        let inner = if acc
-            && tracked_struct_ids.is_empty()
             && cycle_heads.is_empty()
+            && accumulated.is_empty()
             && iteration.is_initial()
         {
             None
         } else {
             Some(Box::new(QueryRevisionsExtraInner {
-                #[cfg(feature = "accumulator")]
                 accumulated,
                 cycle_heads,
                 tracked_struct_ids: tracked_struct_ids.into_thin_vec(),
@@ -512,7 +412,6 @@ impl QueryRevisionsExtra {
 
 #[derive(Debug)]
 struct QueryRevisionsExtraInner {
-    #[cfg(feature = "accumulator")]
     accumulated: AccumulatedMap,
 
     /// The ids of tracked structs created by this query.
@@ -551,18 +450,15 @@ impl QueryRevisionsExtraInner {
     #[cfg(feature = "salsa_unstable")]
     fn allocation_size(&self) -> usize {
         let QueryRevisionsExtraInner {
-            #[cfg(feature = "accumulator")]
             accumulated,
             tracked_struct_ids,
             cycle_heads,
             iteration: _,
         } = self;
 
-        #[cfg(feature = "accumulator")]
+        accumulated.allocation_size()
-        let b = accumulated.allocation_size();
+            + cycle_heads.allocation_size()
-        #[cfg(not(feature = "accumulator"))]
+            + std::mem::size_of_val(tracked_struct_ids.as_slice())
-        let b = 0;
-        b + cycle_heads.allocation_size() + std::mem::size_of_val(tracked_struct_ids.as_slice())
     }
 }
 
@@ -573,7 +469,7 @@ const _: [(); std::mem::size_of::<QueryRevisions>()] = [(); std::mem::size_of::<
 #[cfg(not(feature = "shuttle"))]
 #[cfg(target_pointer_width = "64")]
 const _: [(); std::mem::size_of::<QueryRevisionsExtraInner>()] =
-    [(); std::mem::size_of::<[usize; if cfg!(feature = "accumulator") { 7 } else { 3 }]>()];
+    [(); std::mem::size_of::<[usize; 7]>()];
 
 impl QueryRevisions {
     pub(crate) fn fixpoint_initial(query: DatabaseKeyIndex) -> Self {
@@ -581,11 +477,9 @@ impl QueryRevisions {
             changed_at: Revision::start(),
             durability: Durability::MAX,
             origin: QueryOrigin::fixpoint_initial(),
-            #[cfg(feature = "accumulator")]
             accumulated_inputs: Default::default(),
             verified_final: AtomicBool::new(false),
             extra: QueryRevisionsExtra::new(
-                #[cfg(feature = "accumulator")]
                 AccumulatedMap::default(),
                 IdentityMap::default(),
                 CycleHeads::initial(query),
@@ -595,7 +489,6 @@ impl QueryRevisions {
     }
 
     /// Returns a reference to the `AccumulatedMap` for this query, or `None` if the map is empty.
-    #[cfg(feature = "accumulator")]
     pub(crate) fn accumulated(&self) -> Option<&AccumulatedMap> {
         self.extra
             .0
@@ -627,7 +520,6 @@ impl QueryRevisions {
             Some(extra) => extra.cycle_heads = cycle_heads,
             None => {
                 self.extra = QueryRevisionsExtra::new(
-                    #[cfg(feature = "accumulator")]
                     AccumulatedMap::default(),
                     IdentityMap::default(),
                     cycle_heads,
@@ -698,7 +590,6 @@ pub enum QueryOriginRef<'a> {
 impl<'a> QueryOriginRef<'a> {
     /// Indices for queries *read* by this query
     #[inline]
-    #[cfg(feature = "accumulator")]
     pub(crate) fn inputs(self) -> impl DoubleEndedIterator<Item = DatabaseKeyIndex> + use<'a> {
         let opt_edges = match self {
             QueryOriginRef::Derived(edges) | QueryOriginRef::DerivedUntracked(edges) => Some(edges),
@@ -863,11 +754,7 @@ impl QueryOrigin {
             QueryOriginKind::Assigned => {
                 // SAFETY: `data.index` is initialized when the tag is `QueryOriginKind::Assigned`.
                 let index = unsafe { self.data.index };
-
+                let ingredient_index = IngredientIndex::from(self.metadata as usize);
-                // SAFETY: `metadata` is initialized from a valid `IngredientIndex` when the tag
-                // is `QueryOriginKind::Assigned`.
-                let ingredient_index = unsafe { IngredientIndex::new_unchecked(self.metadata) };
-
                 QueryOriginRef::Assigned(DatabaseKeyIndex::new(ingredient_index, index))
             }
 
@@ -991,7 +878,6 @@ pub enum QueryEdgeKind {
 /// Returns the (tracked) inputs that were executed in computing this memoized value.
 ///
 /// These will always be in execution order.
-#[cfg(feature = "accumulator")]
 pub(crate) fn input_edges(
     input_outputs: &[QueryEdge],
 ) -> impl DoubleEndedIterator<Item = DatabaseKeyIndex> + use<'_> {
@@ -1027,18 +913,15 @@ pub(crate) struct ActiveQueryGuard<'me> {
 impl ActiveQueryGuard<'_> {
     /// Initialize the tracked struct ids with the values from the prior execution.
     pub(crate) fn seed_tracked_struct_ids(&self, tracked_struct_ids: &[(Identity, Id)]) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.local_state.with_query_stack_mut(|stack| {
-        unsafe {
+            #[cfg(debug_assertions)]
-            self.local_state.with_query_stack_unchecked_mut(|stack| {
+            assert_eq!(stack.len(), self.push_len);
-                #[cfg(debug_assertions)]
+            let frame = stack.last_mut().unwrap();
-                assert_eq!(stack.len(), self.push_len);
+            assert!(frame.tracked_struct_ids().is_empty());
-                let frame = stack.last_mut().unwrap();
+            frame
-                assert!(frame.tracked_struct_ids().is_empty());
+                .tracked_struct_ids_mut()
-                frame
+                .clone_from_slice(tracked_struct_ids);
-                    .tracked_struct_ids_mut()
+        })
-                    .clone_from_slice(tracked_struct_ids);
-            })
-        }
     }
 
     /// Append the given `outputs` to the query's output list.
@@ -1051,29 +934,23 @@ impl ActiveQueryGuard<'_> {
             QueryOriginRef::DerivedUntracked(_)
         );
 
-        // SAFETY: We do not access the query stack reentrantly.
+        self.local_state.with_query_stack_mut(|stack| {
-        unsafe {
+            #[cfg(debug_assertions)]
-            self.local_state.with_query_stack_unchecked_mut(|stack| {
+            assert_eq!(stack.len(), self.push_len);
-                #[cfg(debug_assertions)]
+            let frame = stack.last_mut().unwrap();
-                assert_eq!(stack.len(), self.push_len);
+            frame.seed_iteration(durability, changed_at, edges, untracked_read);
-                let frame = stack.last_mut().unwrap();
+        })
-                frame.seed_iteration(durability, changed_at, edges, untracked_read);
-            })
-        }
     }
 
     /// Invoked when the query has successfully completed execution.
     fn complete(self) -> QueryRevisions {
-        // SAFETY: We do not access the query stack reentrantly.
+        let query = self.local_state.with_query_stack_mut(|stack| {
-        let query = unsafe {
+            stack.pop_into_revisions(
-            self.local_state.with_query_stack_unchecked_mut(|stack| {
+                self.database_key_index,
-                stack.pop_into_revisions(
+                #[cfg(debug_assertions)]
-                    self.database_key_index,
+                self.push_len,
-                    #[cfg(debug_assertions)]
+            )
-                    self.push_len,
+        });
-                )
-            })
-        };
         std::mem::forget(self);
         query
     }
@@ -1089,15 +966,12 @@ impl ActiveQueryGuard<'_> {
 
 impl Drop for ActiveQueryGuard<'_> {
     fn drop(&mut self) {
-        // SAFETY: We do not access the query stack reentrantly.
+        self.local_state.with_query_stack_mut(|stack| {
-        unsafe {
+            stack.pop(
-            self.local_state.with_query_stack_unchecked_mut(|stack| {
+                self.database_key_index,
-                stack.pop(
+                #[cfg(debug_assertions)]
-                    self.database_key_index,
+                self.push_len,
-                    #[cfg(debug_assertions)]
+            );
-                    self.push_len,
+        });
-                );
-            })
-        };
     }
 }

tests/accumulate-chain.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Test that when having nested tracked functions
 //! we don't drop any values when accumulating.
 

tests/accumulate-custom-debug.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 mod common;
 
 use expect_test::expect;

tests/accumulate-dag.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 mod common;
 
 use expect_test::expect;

tests/accumulate-execution-order.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Demonstrates that accumulation is done in the order
 //! in which things were originally executed.
 

tests/accumulate-from-tracked-fn.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Accumulate values from within a tracked function.
 //! Then mutate the values so that the tracked function re-executes.
 //! Check that we accumulate the appropriate, new values.

tests/accumulate-no-duplicates.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Test that we don't get duplicate accumulated values
 
 mod common;

tests/accumulate-reuse-workaround.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Demonstrates the workaround of wrapping calls to
 //! `accumulated` in a tracked function to get better
 //! reuse.

tests/accumulate-reuse.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Accumulator re-use test.
 //!
 //! Tests behavior when a query's only inputs

tests/accumulate.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 mod common;
 use common::{LogDatabase, LoggerDatabase};
 use expect_test::expect;

tests/accumulated_backdate.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 //! Tests that accumulated values are correctly accounted for
 //! when backdating a value.
 

tests/backtrace.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 use expect_test::expect;
 use salsa::{Backtrace, Database, DatabaseImpl};
 use test_log::test;
@@ -73,15 +71,15 @@ fn backtrace_works() {
     expect![[r#"
         query stacktrace:
            0: query_e(Id(0))
-                     at tests/backtrace.rs:32
+                     at tests/backtrace.rs:30
            1: query_d(Id(0))
-                     at tests/backtrace.rs:27
+                     at tests/backtrace.rs:25
            2: query_c(Id(0))
-                     at tests/backtrace.rs:22
+                     at tests/backtrace.rs:20
            3: query_b(Id(0))
-                     at tests/backtrace.rs:17
+                     at tests/backtrace.rs:15
            4: query_a(Id(0))
-                     at tests/backtrace.rs:12
+                     at tests/backtrace.rs:10
     "#]]
     .assert_eq(&backtrace);
 
@@ -89,15 +87,15 @@ fn backtrace_works() {
     expect![[r#"
         query stacktrace:
            0: query_e(Id(1)) -> (R1, Durability::LOW)
-                     at tests/backtrace.rs:32
+                     at tests/backtrace.rs:30
            1: query_d(Id(1)) -> (R1, Durability::HIGH)
-                     at tests/backtrace.rs:27
+                     at tests/backtrace.rs:25
            2: query_c(Id(1)) -> (R1, Durability::HIGH)
-                     at tests/backtrace.rs:22
+                     at tests/backtrace.rs:20
            3: query_b(Id(1)) -> (R1, Durability::HIGH)
-                     at tests/backtrace.rs:17
+                     at tests/backtrace.rs:15
            4: query_a(Id(1)) -> (R1, Durability::HIGH)
-                     at tests/backtrace.rs:12
+                     at tests/backtrace.rs:10
     "#]]
     .assert_eq(&backtrace);
 
@@ -105,12 +103,12 @@ fn backtrace_works() {
     expect![[r#"
         query stacktrace:
            0: query_e(Id(2))
-                     at tests/backtrace.rs:32
+                     at tests/backtrace.rs:30
            1: query_cycle(Id(2))
-                     at tests/backtrace.rs:45
+                     at tests/backtrace.rs:43
                      cycle heads: query_cycle(Id(2)) -> IterationCount(0)
            2: query_f(Id(2))
-                     at tests/backtrace.rs:40
+                     at tests/backtrace.rs:38
     "#]]
     .assert_eq(&backtrace);
 
@@ -118,12 +116,12 @@ fn backtrace_works() {
     expect![[r#"
         query stacktrace:
            0: query_e(Id(3)) -> (R1, Durability::LOW)
-                     at tests/backtrace.rs:32
+                     at tests/backtrace.rs:30
            1: query_cycle(Id(3)) -> (R1, Durability::HIGH, iteration = IterationCount(0))
-                     at tests/backtrace.rs:45
+                     at tests/backtrace.rs:43
                      cycle heads: query_cycle(Id(3)) -> IterationCount(0)
            2: query_f(Id(3)) -> (R1, Durability::HIGH)
-                     at tests/backtrace.rs:40
+                     at tests/backtrace.rs:38
     "#]]
     .assert_eq(&backtrace);
 }

tests/check_auto_traits.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test that auto trait impls exist as expected.
 
 use std::panic::UnwindSafe;

tests/common/mod.rs

@@ -34,10 +34,6 @@ pub trait LogDatabase: HasLogger + Database {
         self.logger().logs.lock().unwrap().push(string);
     }
 
-    fn clear_logs(&self) {
-        std::mem::take(&mut *self.logger().logs.lock().unwrap());
-    }
-
     /// Asserts what the (formatted) logs should look like,
     /// clearing the logged events. This takes `&mut self` because
     /// it is meant to be run from outside any tracked functions.

tests/compile-fail/input_struct_incompatibles.rs

@@ -25,4 +25,7 @@ struct InputWithTrackedField {
     field: u32,
 }
 
+#[salsa::input(heap_size = size)]
+struct InputWithHeapSize(u32);
+
 fn main() {}

tests/compile-fail/input_struct_incompatibles.stderr

@@ -47,6 +47,12 @@ error: `#[tracked]` cannot be used with `#[salsa::input]`
 25 | |     field: u32,
    | |______________^
 
+error: `heap_size` option not allowed here
+  --> tests/compile-fail/input_struct_incompatibles.rs:28:16
+   |
+28 | #[salsa::input(heap_size = size)]
+   |                ^^^^^^^^^
+
 error: cannot find attribute `tracked` in this scope
   --> tests/compile-fail/input_struct_incompatibles.rs:24:7
    |

tests/compile-fail/interned_struct_incompatibles.rs

@@ -39,4 +39,9 @@ struct InternedWithZeroRevisions {
     field: u32,
 }
 
+#[salsa::interned(heap_size = size)]
+struct AccWithHeapSize {
+    field: u32,
+}
+
 fn main() {}

tests/compile-fail/interned_struct_incompatibles.stderr

@@ -41,6 +41,12 @@ error: `#[tracked]` cannot be used with `#[salsa::interned]`
 34 | |     field: u32,
    | |______________^
 
+error: `heap_size` option not allowed here
+  --> tests/compile-fail/interned_struct_incompatibles.rs:42:19
+   |
+42 | #[salsa::interned(heap_size = size)]
+   |                   ^^^^^^^^^
+
 error: cannot find attribute `tracked` in this scope
   --> tests/compile-fail/interned_struct_incompatibles.rs:33:7
    |

tests/compile-fail/tracked_struct_incompatibles.rs

@@ -33,4 +33,9 @@ struct TrackedStructWithRevisions {
     field: u32,
 }
 
+#[salsa::tracked(heap_size = size)]
+struct TrackedStructWithHeapSize {
+    field: u32,
+}
+
 fn main() {}

tests/compile-fail/tracked_struct_incompatibles.stderr

@@ -39,3 +39,9 @@ error: `revisions` option not allowed here
    |
 31 | #[salsa::tracked(revisions = 12)]
    |                  ^^^^^^^^^
+
+error: `heap_size` option not allowed here
+  --> tests/compile-fail/tracked_struct_incompatibles.rs:36:18
+   |
+36 | #[salsa::tracked(heap_size = size)]
+   |                  ^^^^^^^^^

tests/compile_fail.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 #[rustversion::all(stable, since(1.84))]
 #[test]
 fn compile_fail() {

tests/cycle.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test cases for fixpoint iteration cycle resolution.
 //!
 //! These test cases use a generic query setup that allows constructing arbitrary dependency
@@ -1025,42 +1023,3 @@ fn repeat_provisional_query() {
             "salsa_event(WillExecute { database_key: min_panic(Id(2)) })",
         ]"#]]);
 }
-
-#[test]
-fn repeat_provisional_query_incremental() {
-    let mut db = ExecuteValidateLoggerDatabase::default();
-    let a_in = Inputs::new(&db, vec![]);
-    let b_in = Inputs::new(&db, vec![]);
-    let c_in = Inputs::new(&db, vec![]);
-    let a = Input::MinIterate(a_in);
-    let b = Input::MinPanic(b_in);
-    let c = Input::MinPanic(c_in);
-    a_in.set_inputs(&mut db).to(vec![value(59), b.clone()]);
-    b_in.set_inputs(&mut db)
-        .to(vec![value(60), c.clone(), c.clone(), c]);
-    c_in.set_inputs(&mut db).to(vec![a.clone()]);
-
-    a.assert_value(&db, 59);
-
-    db.clear_logs();
-
-    c_in.set_inputs(&mut db).to(vec![a.clone()]);
-
-    a.assert_value(&db, 59);
-
-    // `min_panic(Id(2)) should only twice:
-    // * Once before iterating
-    // * Once as part of iterating
-    //
-    // If it runs more than once before iterating, than this suggests that
-    // `validate_same_iteration` incorrectly returns `false`.
-    db.assert_logs(expect![[r#"
-        [
-            "salsa_event(WillExecute { database_key: min_panic(Id(2)) })",
-            "salsa_event(WillExecute { database_key: min_panic(Id(1)) })",
-            "salsa_event(WillExecute { database_key: min_iterate(Id(0)) })",
-            "salsa_event(WillIterateCycle { database_key: min_iterate(Id(0)), iteration_count: IterationCount(1), fell_back: false })",
-            "salsa_event(WillExecute { database_key: min_panic(Id(1)) })",
-            "salsa_event(WillExecute { database_key: min_panic(Id(2)) })",
-        ]"#]]);
-}

tests/cycle_accumulate.rs

@@ -1,5 +1,3 @@
-#![cfg(all(feature = "inventory", feature = "accumulator"))]
-
 use std::collections::HashSet;
 
 mod common;

tests/cycle_fallback_immediate.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! It is possible to omit the `cycle_fn`, only specifying `cycle_result` in which case
 //! an immediate fallback value is used as the cycle handling opposed to doing a fixpoint resolution.
 

tests/cycle_initial_call_back_into_cycle.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Calling back into the same cycle from your cycle initial function will trigger another cycle.
 
 #[salsa::tracked]

tests/cycle_initial_call_query.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! It's possible to call a Salsa query from within a cycle initial fn.
 
 #[salsa::tracked]

tests/cycle_maybe_changed_after.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Tests for incremental validation for queries involved in a cycle.
 mod common;
 

tests/cycle_output.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test tracked struct output from a query in a cycle.
 mod common;
 use common::{HasLogger, LogDatabase, Logger};

tests/cycle_recovery_call_back_into_cycle.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Calling back into the same cycle from your cycle recovery function _can_ work out, as long as
 //! the overall cycle still converges.
 

tests/cycle_recovery_call_query.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! It's possible to call a Salsa query from within a cycle recovery fn.
 
 #[salsa::tracked]

tests/cycle_regression_455.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 use salsa::{Database, Setter};
 
 #[salsa::tracked]

tests/cycle_result_dependencies.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 use salsa::{Database, Setter};
 
 #[salsa::input]

tests/cycle_tracked.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Tests for cycles where the cycle head is stored on a tracked struct
 //! and that tracked struct is freed in a later revision.
 

tests/cycle_tracked_own_input.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test for cycle handling where a tracked struct created in the first revision
 //! is stored in the final value of the cycle but isn't recreated in the second
 //! iteration of the creating query.

tests/dataflow.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test case for fixpoint iteration cycle resolution.
 //!
 //! This test case is intended to simulate a (very simplified) version of a real dataflow analysis

tests/debug.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test that `DeriveWithDb` is correctly derived.
 
 use expect_test::expect;

tests/debug_db_contents.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 #[salsa::interned(debug)]
 struct InternedStruct<'db> {
     name: String,
@@ -22,15 +20,14 @@ fn tracked_fn(db: &dyn salsa::Database, input: InputStruct) -> TrackedStruct<'_>
 
 #[test]
 fn execute() {
-    use salsa::plumbing::ZalsaDatabase;
     let db = salsa::DatabaseImpl::new();
 
     let _ = InternedStruct::new(&db, "Salsa".to_string());
     let _ = InternedStruct::new(&db, "Salsa2".to_string());
 
     // test interned structs
-    let interned = InternedStruct::ingredient(db.zalsa())
+    let interned = InternedStruct::ingredient(&db)
-        .entries(db.zalsa())
+        .entries(&db)
         .collect::<Vec<_>>();
 
     assert_eq!(interned.len(), 2);
@@ -41,7 +38,7 @@ fn execute() {
     let input = InputStruct::new(&db, 22);
 
     let inputs = InputStruct::ingredient(&db)
-        .entries(db.zalsa())
+        .entries(&db)
         .collect::<Vec<_>>();
 
     assert_eq!(inputs.len(), 1);
@@ -51,7 +48,7 @@ fn execute() {
     let computed = tracked_fn(&db, input).field(&db);
     assert_eq!(computed, 44);
     let tracked = TrackedStruct::ingredient(&db)
-        .entries(db.zalsa())
+        .entries(&db)
         .collect::<Vec<_>>();
 
     assert_eq!(tracked.len(), 1);

tests/deletion-cascade.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Delete cascade:
 //!
 //! * when we delete memoized data, also delete outputs from that data

tests/deletion-drops.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Basic deletion test:
 //!
 //! * entities not created in a revision are deleted, as is any memoized data keyed on them.

tests/deletion.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Basic deletion test:
 //!
 //! * entities not created in a revision are deleted, as is any memoized data keyed on them.

tests/derive_update.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test that the `Update` derive works as expected
 
 #[derive(salsa::Update)]

tests/durability.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Tests that code using the builder's durability methods compiles.
 
 use salsa::{Database, Durability, Setter};

tests/elided-lifetime-in-tracked-fn.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test that a `tracked` fn on a `salsa::input`
 //! compiles and executes successfully.
 

tests/expect_reuse_field_x_of_a_tracked_struct_changes_but_fn_depends_on_field_y.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test that if field X of a tracked struct changes but not field Y,
 //! functions that depend on X re-execute, but those depending only on Y do not
 //! compiles and executes successfully.

tests/expect_reuse_field_x_of_an_input_changes_but_fn_depends_on_field_y.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "inventory")]
-
 //! Test that if field X of an input changes but not field Y,
 //! functions that depend on X re-execute, but those depending only on Y do not
 //! compiles and executes successfully.