Merge 'Fix: allow page_size=65536' from meteorgan

Since `page_size` in `DatabaseHeader` can be 1 representing 65526 bytes,
it can't be used it directly.  Additionally, we should use `u32` instead
of `u16` or `usize` in other contexts.

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #1411

core/lib.rs

@@ -96,7 +96,7 @@ pub struct Database {
     header: Arc<SpinLock<DatabaseHeader>>,
     db_file: Arc<dyn DatabaseStorage>,
     io: Arc<dyn IO>,
-    page_size: u16,
+    page_size: u32,
     // Shared structures of a Database are the parts that are common to multiple threads that might
     // create DB connections.
     shared_page_cache: Arc<RwLock<DumbLruPageCache>>,
@@ -126,7 +126,7 @@ impl Database {
         // ensure db header is there
         io.run_once()?;
 
-        let page_size = db_header.lock().page_size;
+        let page_size = db_header.lock().get_page_size();
         let wal_path = format!("{}-wal", path);
         let shared_wal = WalFileShared::open_shared(&io, wal_path.as_str(), page_size)?;
 
@@ -181,7 +181,7 @@ impl Database {
 
         let wal = Rc::new(RefCell::new(WalFile::new(
             self.io.clone(),
-            self.page_size as usize,
+            self.page_size,
             self.shared_wal.clone(),
             buffer_pool.clone(),
         )));
@@ -244,7 +244,7 @@ pub fn maybe_init_database_file(file: &Arc<dyn File>, io: &Arc<dyn IO>) -> Resul
         let db_header = DatabaseHeader::default();
         let page1 = allocate_page(
             1,
-            &Rc::new(BufferPool::new(db_header.page_size as usize)),
+            &Rc::new(BufferPool::new(db_header.get_page_size() as usize)),
             DATABASE_HEADER_SIZE,
         );
         {
@@ -256,7 +256,7 @@ pub fn maybe_init_database_file(file: &Arc<dyn File>, io: &Arc<dyn IO>) -> Resul
                 &page1,
                 storage::sqlite3_ondisk::PageType::TableLeaf,
                 DATABASE_HEADER_SIZE,
-                db_header.page_size - db_header.reserved_space as u16,
+                (db_header.get_page_size() - db_header.reserved_space as u32) as u16,
             );
 
             let contents = page1.get().contents.as_mut().unwrap();

core/storage/btree.rs

@@ -5493,15 +5493,15 @@ mod tests {
 
     fn empty_btree() -> (Rc<Pager>, usize) {
         let db_header = DatabaseHeader::default();
-        let page_size = db_header.page_size as usize;
+        let page_size = db_header.get_page_size();
 
         #[allow(clippy::arc_with_non_send_sync)]
         let io: Arc<dyn IO> = Arc::new(MemoryIO::new());
         let io_file = io.open_file("test.db", OpenFlags::Create, false).unwrap();
         let db_file = Arc::new(DatabaseFile::new(io_file));
 
-        let buffer_pool = Rc::new(BufferPool::new(db_header.page_size as usize));
-        let wal_shared = WalFileShared::open_shared(&io, "test.wal", db_header.page_size).unwrap();
+        let buffer_pool = Rc::new(BufferPool::new(page_size as usize));
+        let wal_shared = WalFileShared::open_shared(&io, "test.wal", page_size).unwrap();
         let wal_file = WalFile::new(io.clone(), page_size, wal_shared, buffer_pool.clone());
         let wal = Rc::new(RefCell::new(wal_file));
 
@@ -5864,7 +5864,7 @@ mod tests {
     fn setup_test_env(database_size: u32) -> (Rc<Pager>, Arc<SpinLock<DatabaseHeader>>) {
         let page_size = 512;
         let mut db_header = DatabaseHeader::default();
-        db_header.page_size = page_size;
+        db_header.update_page_size(page_size);
         db_header.database_size = database_size;
         let db_header = Arc::new(SpinLock::new(db_header));
 
@@ -5896,7 +5896,7 @@ mod tests {
         let wal_shared = WalFileShared::open_shared(&io, "test.wal", page_size).unwrap();
         let wal = Rc::new(RefCell::new(WalFile::new(
             io.clone(),
-            page_size as usize,
+            page_size,
             wal_shared,
             buffer_pool.clone(),
         )));
@@ -5936,7 +5936,7 @@ mod tests {
             let drop_fn = Rc::new(|_buf| {});
             #[allow(clippy::arc_with_non_send_sync)]
             let buf = Arc::new(RefCell::new(Buffer::allocate(
-                db_header.lock().page_size as usize,
+                db_header.lock().get_page_size() as usize,
                 drop_fn,
             )));
             let write_complete = Box::new(|_| {});

core/storage/pager.rs

@@ -257,7 +257,7 @@ impl Pager {
     /// In other words, if the page size is 512, then the reserved space size cannot exceed 32.
     pub fn usable_space(&self) -> usize {
         let db_header = self.db_header.lock();
-        (db_header.page_size - db_header.reserved_space as u16) as usize
+        (db_header.get_page_size() - db_header.reserved_space as u32) as usize
     }
 
     #[inline(always)]
@@ -685,7 +685,7 @@ impl Pager {
 
     pub fn usable_size(&self) -> usize {
         let db_header = self.db_header.lock();
-        (db_header.page_size - db_header.reserved_space as u16) as usize
+        (db_header.get_page_size() - db_header.reserved_space as u32) as usize
     }
 }
 

core/storage/sqlite3_ondisk.rs

@@ -65,11 +65,19 @@ pub const DATABASE_HEADER_SIZE: usize = 100;
 // DEFAULT_CACHE_SIZE negative values mean that we store the amount of pages a XKiB of memory can hold.
 // We can calculate "real" cache size by diving by page size.
 const DEFAULT_CACHE_SIZE: i32 = -2000;
-// The size of db page in bytes.
-const DEFAULT_PAGE_SIZE: u16 = 4096;
+
 // Minimum number of pages that cache can hold.
 pub const MIN_PAGE_CACHE_SIZE: usize = 10;
 
+/// The minimum page size in bytes.
+const MIN_PAGE_SIZE: u32 = 512;
+
+/// The maximum page size in bytes.
+const MAX_PAGE_SIZE: u32 = 65536;
+
+/// The default page size in bytes.
+const DEFAULT_PAGE_SIZE: u16 = 4096;
+
 /// The database header.
 /// The first 100 bytes of the database file comprise the database file header.
 /// The database file header is divided into fields as shown by the table below.
@@ -81,7 +89,7 @@ pub struct DatabaseHeader {
 
     /// The database page size in bytes. Must be a power of two between 512 and 32768 inclusive,
     /// or the value 1 representing a page size of 65536.
-    pub page_size: u16,
+    page_size: u16,
 
     /// File format write version. 1 for legacy; 2 for WAL.
     write_version: u8,
@@ -172,7 +180,7 @@ pub struct WalHeader {
     /// WAL format version. Currently 3007000
     pub file_format: u32,
 
-    /// Database page size in bytes. Power of two between 512 and 32768 inclusive
+    /// Database page size in bytes. Power of two between 512 and 65536 inclusive
     pub page_size: u32,
 
     /// Checkpoint sequence number. Increases with each checkpoint
@@ -247,6 +255,28 @@ impl Default for DatabaseHeader {
     }
 }
 
+impl DatabaseHeader {
+    pub fn update_page_size(&mut self, size: u32) {
+        if !(MIN_PAGE_SIZE..=MAX_PAGE_SIZE).contains(&size) || (size & (size - 1) != 0) {
+            return;
+        }
+
+        self.page_size = if size == MAX_PAGE_SIZE {
+            1u16
+        } else {
+            size as u16
+        };
+    }
+
+    pub fn get_page_size(&self) -> u32 {
+        if self.page_size == 1 {
+            MAX_PAGE_SIZE
+        } else {
+            self.page_size as u32
+        }
+    }
+}
+
 pub fn begin_read_database_header(
     db_file: Arc<dyn DatabaseStorage>,
 ) -> Result<Arc<SpinLock<DatabaseHeader>>> {

core/storage/wal.rs

@@ -246,7 +246,7 @@ pub struct WalFile {
 
     sync_state: RefCell<SyncState>,
     syncing: Rc<RefCell<bool>>,
-    page_size: usize,
+    page_size: u32,
 
     shared: Arc<UnsafeCell<WalFileShared>>,
     ongoing_checkpoint: OngoingCheckpoint,
@@ -688,7 +688,7 @@ impl Wal for WalFile {
 impl WalFile {
     pub fn new(
         io: Arc<dyn IO>,
-        page_size: usize,
+        page_size: u32,
         shared: Arc<UnsafeCell<WalFileShared>>,
         buffer_pool: Rc<BufferPool>,
     ) -> Self {
@@ -728,7 +728,7 @@ impl WalFile {
     fn frame_offset(&self, frame_id: u64) -> usize {
         assert!(frame_id > 0, "Frame ID must be 1-based");
         let page_size = self.page_size;
-        let page_offset = (frame_id - 1) * (page_size + WAL_FRAME_HEADER_SIZE) as u64;
+        let page_offset = (frame_id - 1) * (page_size + WAL_FRAME_HEADER_SIZE as u32) as u64;
         let offset = WAL_HEADER_SIZE as u64 + page_offset;
         offset as usize
     }
@@ -743,7 +743,7 @@ impl WalFileShared {
     pub fn open_shared(
         io: &Arc<dyn IO>,
         path: &str,
-        page_size: u16,
+        page_size: u32,
     ) -> Result<Arc<UnsafeCell<WalFileShared>>> {
         let file = io.open_file(path, crate::io::OpenFlags::Create, false)?;
         let header = if file.size()? > 0 {
@@ -764,7 +764,7 @@ impl WalFileShared {
             let mut wal_header = WalHeader {
                 magic,
                 file_format: 3007000,
-                page_size: page_size as u32,
+                page_size,
                 checkpoint_seq: 0, // TODO implement sequence number
                 salt_1: io.generate_random_number() as u32,
                 salt_2: io.generate_random_number() as u32,

core/translate/pragma.rs

@@ -261,7 +261,7 @@ fn query_pragma(
             program.emit_result_row(register, 1);
         }
         PragmaName::PageSize => {
-            program.emit_int(database_header.lock().page_size.into(), register);
+            program.emit_int(database_header.lock().get_page_size().into(), register);
             program.emit_result_row(register, 1);
         }
     }

core/vdbe/execute.rs

@@ -4548,7 +4548,7 @@ pub fn op_open_ephemeral(
     let db_file = Arc::new(FileMemoryStorage::new(file));
 
     let db_header = Pager::begin_open(db_file.clone())?;
-    let buffer_pool = Rc::new(BufferPool::new(db_header.lock().page_size as usize));
+    let buffer_pool = Rc::new(BufferPool::new(db_header.lock().get_page_size() as usize));
     let page_cache = Arc::new(RwLock::new(DumbLruPageCache::new(10)));
 
     let pager = Rc::new(Pager::finish_open(