mirrors/limbo
1
0
Fork 0
mirror of https://github.com/tursodatabase/limbo.git synced 2025-08-04 18:18:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

Fix detachment of nodes in lru cache.

Browse source
This commit is contained in:
Pere Diaz Bou 2025-03-12 15:39:03 +01:00
parent be3badc1f3
commit deaff6c1ec
3 changed files with 209 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

18
Cargo.lock generated
View file

@ -48,6 +48,12 @@ dependencies = [
"equator",
]
[[package]]
name = "allocator-api2"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
name = "anarchist-readable-name-generator-lib"
version = "0.1.2"
@ -1083,6 +1089,8 @@ version = "0.15.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
]
@ -1658,6 +1666,7 @@ dependencies = [
"limbo_sqlite3_parser",
"limbo_time",
"limbo_uuid",
"lru",
"miette",
"mimalloc",
"parking_lot",
@ -1887,6 +1896,15 @@ version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
[[package]]
name = "lru"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "227748d55f2f0ab4735d87fd623798cb6b664512fe979705f829c9f81c934465"
dependencies = [
"hashbrown",
]
[[package]]
name = "matchers"
version = "0.1.0"

1
core/Cargo.toml
View file

@ -97,6 +97,7 @@ rand = "0.8.5" # Required for quickcheck
rand_chacha = "0.9.0"
env_logger = "0.11.6"
test-log = { version = "0.2.17", features = ["trace"] }
lru = "0.13.0"
[[bench]]
name = "benchmark"

226
core/storage/page_cache.rs
View file

@ -62,37 +62,34 @@ impl DumbLruPageCache {
pub fn insert(&mut self, key: PageCacheKey, value: PageRef) {
self._delete(key.clone(), false);
debug!("cache_insert(key={:?})", key);
let mut entry = Box::new(PageCacheEntry {
let entry = Box::new(PageCacheEntry {
key: key.clone(),
next: None,
prev: None,
page: value,
});
self.touch(&mut entry);
let ptr_raw = Box::into_raw(entry);
let ptr = unsafe { ptr_raw.as_mut().unwrap().as_non_null() };
self.touch(ptr);
if self.map.borrow().len() >= self.capacity {
self.map.borrow_mut().insert(key, ptr);
if self.len() > self.capacity {
self.pop_if_not_dirty();
}
let b = Box::into_raw(entry);
let as_non_null = NonNull::new(b).unwrap();
self.map.borrow_mut().insert(key, as_non_null);
}
pub fn delete(&mut self, key: PageCacheKey) {
debug!("cache_delete(key={:?})", key);
self._delete(key, true)
}
pub fn _delete(&mut self, key: PageCacheKey, clean_page: bool) {
debug!("cache_delete(key={:?}, clean={})", key, clean_page);
let ptr = self.map.borrow_mut().remove(&key);
if ptr.is_none() {
return;
}
let mut ptr = ptr.unwrap();
{
let ptr = unsafe { ptr.as_mut() };
self.detach(ptr, clean_page);
}
let ptr = ptr.unwrap();
self.detach(ptr, clean_page);
unsafe { std::ptr::drop_in_place(ptr.as_ptr()) };
}
@ -103,13 +100,18 @@ impl DumbLruPageCache {
}
pub fn get(&mut self, key: &PageCacheKey) -> Option<PageRef> {
self.peek(key, true)
}
/// Get page without promoting entry
pub fn peek(&mut self, key: &PageCacheKey, touch: bool) -> Option<PageRef> {
debug!("cache_get(key={:?})", key);
let ptr = self.get_ptr(key);
ptr?;
let ptr = unsafe { ptr.unwrap().as_mut() };
let page = ptr.page.clone();
//self.detach(ptr);
self.touch(ptr);
let mut ptr = self.get_ptr(key)?;
let page = unsafe { ptr.as_mut().page.clone() };
if touch {
self.detach(ptr, false);
self.touch(ptr);
}
Some(page)
}
@ -118,19 +120,17 @@ impl DumbLruPageCache {
todo!();
}
fn detach(&mut self, entry: &mut PageCacheEntry, clean_page: bool) {
let mut current = entry.as_non_null();
fn detach(&mut self, mut entry: NonNull<PageCacheEntry>, clean_page: bool) {
if clean_page {
// evict buffer
let page = &entry.page;
let page = unsafe { &entry.as_mut().page };
page.clear_loaded();
debug!("cleaning up page {}", page.get().id);
let _ = page.get().contents.take();
}
let (next, prev) = unsafe {
let c = current.as_mut();
let c = entry.as_mut();
let next = c.next;
let prev = c.prev;
c.prev = None;
@ -140,9 +140,16 @@ impl DumbLruPageCache {
// detach
match (prev, next) {
(None, None) => {}
(None, Some(_)) => todo!(),
(Some(p), None) => {
(None, None) => {
self.head.replace(None);
self.tail.replace(None);
}
(None, Some(mut n)) => {
unsafe { n.as_mut().prev = None };
self.head.borrow_mut().replace(n);
}
(Some(mut p), None) => {
unsafe { p.as_mut().next = None };
self.tail = RefCell::new(Some(p));
}
(Some(mut p), Some(mut n)) => unsafe {
@ -154,19 +161,20 @@ impl DumbLruPageCache {
};
}
fn touch(&mut self, entry: &mut PageCacheEntry) {
let mut current = entry.as_non_null();
unsafe {
let c = current.as_mut();
c.next = *self.head.borrow();
}
/// inserts into head, assuming we detached first
fn touch(&mut self, mut entry: NonNull<PageCacheEntry>) {
if let Some(mut head) = *self.head.borrow_mut() {
unsafe {
entry.as_mut().next.replace(head);
let head = head.as_mut();
head.prev = Some(current);
head.prev = Some(entry);
}
}
if self.tail.borrow().is_none() {
self.tail.borrow_mut().replace(entry);
}
self.head.borrow_mut().replace(entry);
}
fn pop_if_not_dirty(&mut self) {
@ -174,12 +182,14 @@ impl DumbLruPageCache {
if tail.is_none() {
return;
}
let tail = unsafe { tail.unwrap().as_mut() };
if tail.page.is_dirty() {
let mut tail = tail.unwrap();
let tail_entry = unsafe { tail.as_mut() };
if tail_entry.page.is_dirty() {
// TODO: drop from another clean entry?
return;
}
self.detach(tail, true);
assert!(self.map.borrow_mut().remove(&tail_entry.key).is_some());
}
pub fn clear(&mut self) {
@ -188,4 +198,148 @@ impl DumbLruPageCache {
self.delete(key);
}
}
pub fn print(&mut self) {
println!("page_cache={}", self.map.borrow().len());
println!("page_cache={:?}", self.map.borrow())
}
pub fn len(&self) -> usize {
self.map.borrow().len()
}
}
#[cfg(test)]
mod tests {
use std::{num::NonZeroUsize, sync::Arc};
use lru::LruCache;
use rand_chacha::{
rand_core::{RngCore, SeedableRng},
ChaCha8Rng,
};
use crate::{storage::page_cache::DumbLruPageCache, Page};
use super::PageCacheKey;
#[test]
fn test_page_cache_evict() {
let mut cache = DumbLruPageCache::new(1);
let key1 = insert_page(&mut cache, 1);
let key2 = insert_page(&mut cache, 2);
assert_eq!(cache.get(&key2).unwrap().get().id, 2);
assert!(cache.get(&key1).is_none());
}
#[test]
fn test_page_cache_fuzz() {
let seed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let mut rng = ChaCha8Rng::seed_from_u64(seed);
tracing::info!("super seed: {}", seed);
let max_pages = 10;
let mut cache = DumbLruPageCache::new(10);
let mut lru = LruCache::new(NonZeroUsize::new(10).unwrap());
for _ in 0..10000 {
match rng.next_u64() % 3 {
0 => {
// add
let id_page = rng.next_u64() % max_pages;
let id_frame = rng.next_u64() % max_pages;
let key = PageCacheKey::new(id_page as usize, Some(id_frame));
let page = Arc::new(Page::new(id_page as usize));
// println!("inserting page {:?}", key);
cache.insert(key.clone(), page.clone());
lru.push(key, page);
assert!(cache.len() <= 10);
}
1 => {
// remove
let random = rng.next_u64() % 1 == 0;
let key = if random {
let id_page = rng.next_u64() % max_pages;
let id_frame = rng.next_u64() % max_pages;
let key = PageCacheKey::new(id_page as usize, Some(id_frame));
key
} else {
let i = rng.next_u64() as usize % lru.len();
let key = lru.iter().skip(i).next().unwrap().0.clone();
key
};
// println!("removing page {:?}", key);
lru.pop(&key);
cache.delete(key);
}
2 => {
// cache.print();
// println!("lru={:?}", lru);
// test contents
for (key, page) in &lru {
// println!("getting page {:?}", key);
cache.peek(&key, false).unwrap();
assert_eq!(page.get().id, key.pgno);
}
}
_ => unreachable!(),
}
}
}
#[test]
fn test_page_cache_insert_and_get() {
let mut cache = DumbLruPageCache::new(2);
let key1 = insert_page(&mut cache, 1);
let key2 = insert_page(&mut cache, 2);
assert_eq!(cache.get(&key1).unwrap().get().id, 1);
assert_eq!(cache.get(&key2).unwrap().get().id, 2);
}
#[test]
fn test_page_cache_over_capacity() {
let mut cache = DumbLruPageCache::new(2);
let key1 = insert_page(&mut cache, 1);
let key2 = insert_page(&mut cache, 2);
let key3 = insert_page(&mut cache, 3);
assert!(cache.get(&key1).is_none());
assert_eq!(cache.get(&key2).unwrap().get().id, 2);
assert_eq!(cache.get(&key3).unwrap().get().id, 3);
}
#[test]
fn test_page_cache_delete() {
let mut cache = DumbLruPageCache::new(2);
let key1 = insert_page(&mut cache, 1);
cache.delete(key1.clone());
assert!(cache.get(&key1).is_none());
}
#[test]
fn test_page_cache_clear() {
let mut cache = DumbLruPageCache::new(2);
let key1 = insert_page(&mut cache, 1);
let key2 = insert_page(&mut cache, 2);
cache.clear();
assert!(cache.get(&key1).is_none());
assert!(cache.get(&key2).is_none());
}
fn insert_page(cache: &mut DumbLruPageCache, id: usize) -> PageCacheKey {
let key = PageCacheKey::new(id, None);
let page = Arc::new(Page::new(id));
cache.insert(key.clone(), page.clone());
key
}
#[test]
fn test_page_cache_insert_sequential() {
let mut cache = DumbLruPageCache::new(2);
for i in 0..10000 {
let key = insert_page(&mut cache, i);
// assert_eq!(cache.peek(&key, false).unwrap().get().id, i);
}
}
}