Reorganize ext library and feature gate vfs to more easily prevent wasm build issues

core/Cargo.toml

@@ -15,7 +15,7 @@ path = "lib.rs"
 
 [features]
 default = ["fs", "json", "uuid", "time"]
-fs = []
+fs = ["limbo_ext/vfs"]
 json = ["dep:jsonb", "dep:pest", "dep:pest_derive", "dep:serde", "dep:indexmap"]
 uuid = ["limbo_uuid/static"]
 io_uring = ["dep:io-uring", "rustix/io_uring", "dep:libc"]

core/ext/dynamic.rs

@@ -1,7 +1,13 @@
-use crate::{Connection, LimboError};
+use crate::{
+    ext::{register_aggregate_function, register_scalar_function, register_vtab_module},
+    Connection, LimboError,
+};
 use libloading::{Library, Symbol};
-use limbo_ext::{ExtensionApi, ExtensionApiRef, ExtensionEntryPoint};
-use std::sync::{Arc, Mutex, OnceLock};
+use limbo_ext::{ExtensionApi, ExtensionApiRef, ExtensionEntryPoint, ResultCode, VfsImpl};
+use std::{
+    ffi::{c_char, CString},
+    sync::{Arc, Mutex, OnceLock},
+};
 
 type ExtensionStore = Vec<(Arc<Library>, ExtensionApiRef)>;
 static EXTENSIONS: OnceLock<Arc<Mutex<ExtensionStore>>> = OnceLock::new();
@@ -11,6 +17,17 @@ pub fn get_extension_libraries() -> Arc<Mutex<ExtensionStore>> {
         .clone()
 }
 
+type Vfs = (String, Arc<VfsMod>);
+static VFS_MODULES: OnceLock<Mutex<Vec<Vfs>>> = OnceLock::new();
+
+#[derive(Clone, Debug)]
+pub struct VfsMod {
+    pub ctx: *const VfsImpl,
+}
+
+unsafe impl Send for VfsMod {}
+unsafe impl Sync for VfsMod {}
+
 impl Connection {
     pub fn load_extension<P: AsRef<std::ffi::OsStr>>(&self, path: P) -> crate::Result<()> {
         use limbo_ext::ExtensionApiRef;
@@ -39,3 +56,115 @@ impl Connection {
         }
     }
 }
+
+#[allow(clippy::arc_with_non_send_sync)]
+pub(crate) unsafe extern "C" fn register_vfs(
+    name: *const c_char,
+    vfs: *const VfsImpl,
+) -> ResultCode {
+    if name.is_null() || vfs.is_null() {
+        return ResultCode::Error;
+    }
+    let c_str = unsafe { CString::from_raw(name as *mut i8) };
+    let name_str = match c_str.to_str() {
+        Ok(s) => s.to_string(),
+        Err(_) => return ResultCode::Error,
+    };
+    add_vfs_module(name_str, Arc::new(VfsMod { ctx: vfs }));
+    ResultCode::OK
+}
+
+/// Get pointers to all the vfs extensions that need to be built in at compile time.
+/// any other types that are defined in the same extension will not be registered
+/// until the database file is opened and `register_builtins` is called.
+#[cfg(feature = "fs")]
+#[allow(clippy::arc_with_non_send_sync)]
+pub fn add_builtin_vfs_extensions(
+    api: Option<ExtensionApi>,
+) -> crate::Result<Vec<(String, Arc<VfsMod>)>> {
+    use limbo_ext::VfsInterface;
+
+    let mut vfslist: Vec<*const VfsImpl> = Vec::new();
+    let mut api = match api {
+        None => ExtensionApi {
+            ctx: std::ptr::null_mut(),
+            register_scalar_function,
+            register_aggregate_function,
+            register_vtab_module,
+            vfs_interface: VfsInterface {
+                register_vfs,
+                builtin_vfs: vfslist.as_mut_ptr(),
+                builtin_vfs_count: 0,
+            },
+        },
+        Some(mut api) => {
+            api.vfs_interface.builtin_vfs = vfslist.as_mut_ptr();
+            api
+        }
+    };
+    register_static_vfs_modules(&mut api);
+    let mut vfslist = Vec::with_capacity(api.vfs_interface.builtin_vfs_count as usize);
+    let slice = unsafe {
+        std::slice::from_raw_parts_mut(
+            api.vfs_interface.builtin_vfs,
+            api.vfs_interface.builtin_vfs_count as usize,
+        )
+    };
+    for vfs in slice {
+        if vfs.is_null() {
+            continue;
+        }
+        let vfsimpl = unsafe { &**vfs };
+        let name = unsafe {
+            CString::from_raw(vfsimpl.name as *mut i8)
+                .to_str()
+                .map_err(|_| {
+                    LimboError::ExtensionError("unable to register vfs extension".to_string())
+                })?
+                .to_string()
+        };
+        vfslist.push((
+            name,
+            Arc::new(VfsMod {
+                ctx: vfsimpl as *const _,
+            }),
+        ));
+    }
+    Ok(vfslist)
+}
+
+#[cfg(feature = "fs")]
+fn register_static_vfs_modules(_api: &mut ExtensionApi) {
+    #[cfg(feature = "testvfs")]
+    unsafe {
+        limbo_ext_tests::register_extension_static(_api);
+    }
+}
+
+pub fn add_vfs_module(name: String, vfs: Arc<VfsMod>) {
+    let mut modules = VFS_MODULES
+        .get_or_init(|| Mutex::new(Vec::new()))
+        .lock()
+        .unwrap();
+    if !modules.iter().any(|v| v.0 == name) {
+        modules.push((name, vfs));
+    }
+}
+
+pub fn list_vfs_modules() -> Vec<String> {
+    VFS_MODULES
+        .get_or_init(|| Mutex::new(Vec::new()))
+        .lock()
+        .unwrap()
+        .iter()
+        .map(|v| v.0.clone())
+        .collect()
+}
+
+pub fn get_vfs_modules() -> Vec<Vfs> {
+    VFS_MODULES
+        .get_or_init(|| Mutex::new(Vec::new()))
+        .lock()
+        .unwrap()
+        .clone()
+}

core/ext/mod.rs

@@ -1,22 +1,20 @@
-#[cfg(not(target_family = "wasm"))]
+#[cfg(feature = "fs")]
 mod dynamic;
 #[cfg(all(target_os = "linux", feature = "io_uring"))]
 use crate::UringIO;
-use crate::IO;
-use crate::{function::ExternalFunc, Connection, Database, LimboError};
+use crate::{function::ExternalFunc, Connection, Database, LimboError, IO};
+#[cfg(feature = "fs")]
+pub use dynamic::{add_builtin_vfs_extensions, add_vfs_module, list_vfs_modules, VfsMod};
 use limbo_ext::{
-    ExtensionApi, InitAggFunction, ResultCode, ScalarFunction, VTabKind, VTabModuleImpl, VfsImpl,
+    ExtensionApi, InitAggFunction, ResultCode, ScalarFunction, VTabKind, VTabModuleImpl,
 };
 pub use limbo_ext::{FinalizeFunction, StepFunction, Value as ExtValue, ValueType as ExtValueType};
 use std::{
     ffi::{c_char, c_void, CStr, CString},
     rc::Rc,
-    sync::{Arc, Mutex, OnceLock},
+    sync::Arc,
 };
 type ExternAggFunc = (InitAggFunction, StepFunction, FinalizeFunction);
-type Vfs = (String, Arc<VfsMod>);
-
-static VFS_MODULES: OnceLock<Mutex<Vec<Vfs>>> = OnceLock::new();
 
 #[derive(Clone)]
 pub struct VTabImpl {
@@ -24,15 +22,7 @@ pub struct VTabImpl {
     pub implementation: Rc<VTabModuleImpl>,
 }
 
-#[derive(Clone, Debug)]
-pub struct VfsMod {
-    pub ctx: *const VfsImpl,
-}
-
-unsafe impl Send for VfsMod {}
-unsafe impl Sync for VfsMod {}
-
-unsafe extern "C" fn register_scalar_function(
+pub(crate) unsafe extern "C" fn register_scalar_function(
     ctx: *mut c_void,
     name: *const c_char,
     func: ScalarFunction,
@@ -49,7 +39,7 @@ unsafe extern "C" fn register_scalar_function(
     conn.register_scalar_function_impl(&name_str, func)
 }
 
-unsafe extern "C" fn register_aggregate_function(
+pub(crate) unsafe extern "C" fn register_aggregate_function(
     ctx: *mut c_void,
     name: *const c_char,
     args: i32,
@@ -69,7 +59,7 @@ unsafe extern "C" fn register_aggregate_function(
     conn.register_aggregate_function_impl(&name_str, args, (init_func, step_func, finalize_func))
 }
 
-unsafe extern "C" fn register_module(
+pub(crate) unsafe extern "C" fn register_vtab_module(
     ctx: *mut c_void,
     name: *const c_char,
     module: VTabModuleImpl,
@@ -88,79 +78,7 @@ unsafe extern "C" fn register_module(
     }
     let conn = unsafe { &mut *(ctx as *mut Connection) };
 
-    conn.register_module_impl(&name_str, module, kind)
-}
-
-#[allow(clippy::arc_with_non_send_sync)]
-unsafe extern "C" fn register_vfs(name: *const c_char, vfs: *const VfsImpl) -> ResultCode {
-    if name.is_null() || vfs.is_null() {
-        return ResultCode::Error;
-    }
-    let c_str = unsafe { CString::from_raw(name as *mut i8) };
-    let name_str = match c_str.to_str() {
-        Ok(s) => s.to_string(),
-        Err(_) => return ResultCode::Error,
-    };
-    add_vfs_module(name_str, Arc::new(VfsMod { ctx: vfs }));
-    ResultCode::OK
-}
-
-/// Get pointers to all the vfs extensions that need to be built in at compile time.
-/// any other types that are defined in the same extension will not be registered
-/// until the database file is opened and `register_builtins` is called.
-#[cfg(feature = "fs")]
-#[allow(clippy::arc_with_non_send_sync)]
-pub fn add_builtin_vfs_extensions(
-    api: Option<ExtensionApi>,
-) -> crate::Result<Vec<(String, Arc<VfsMod>)>> {
-    let mut vfslist: Vec<*const VfsImpl> = Vec::new();
-    let mut api = match api {
-        None => ExtensionApi {
-            ctx: std::ptr::null_mut(),
-            register_scalar_function,
-            register_aggregate_function,
-            register_vfs,
-            register_module,
-            builtin_vfs: vfslist.as_mut_ptr(),
-            builtin_vfs_count: 0,
-        },
-        Some(mut api) => {
-            api.builtin_vfs = vfslist.as_mut_ptr();
-            api
-        }
-    };
-    register_static_vfs_modules(&mut api);
-    let mut vfslist = Vec::with_capacity(api.builtin_vfs_count as usize);
-    let slice =
-        unsafe { std::slice::from_raw_parts_mut(api.builtin_vfs, api.builtin_vfs_count as usize) };
-    for vfs in slice {
-        if vfs.is_null() {
-            continue;
-        }
-        let vfsimpl = unsafe { &**vfs };
-        let name = unsafe {
-            CString::from_raw(vfsimpl.name as *mut i8)
-                .to_str()
-                .map_err(|_| {
-                    LimboError::ExtensionError("unable to register vfs extension".to_string())
-                })?
-                .to_string()
-        };
-        vfslist.push((
-            name,
-            Arc::new(VfsMod {
-                ctx: vfsimpl as *const _,
-            }),
-        ));
-    }
-    Ok(vfslist)
-}
-
-fn register_static_vfs_modules(_api: &mut ExtensionApi) {
-    #[cfg(feature = "testvfs")]
-    unsafe {
-        limbo_ext_tests::register_extension_static(_api);
-    }
+    conn.register_vtab_module_impl(&name_str, module, kind)
 }
 
 impl Database {
@@ -172,6 +90,7 @@ impl Database {
         vfs: &str,
     ) -> crate::Result<(Arc<dyn IO>, Arc<Database>)> {
         use crate::{MemoryIO, PlatformIO};
+        use dynamic::get_vfs_modules;
 
         let io: Arc<dyn IO> = match vfs {
             "memory" => Arc::new(MemoryIO::new()),
@@ -215,7 +134,7 @@ impl Connection {
         ResultCode::OK
     }
 
-    fn register_module_impl(
+    fn register_vtab_module_impl(
         &mut self,
         name: &str,
         module: VTabModuleImpl,
@@ -238,10 +157,13 @@ impl Connection {
             ctx: self as *const _ as *mut c_void,
             register_scalar_function,
             register_aggregate_function,
-            register_module,
-            register_vfs,
-            builtin_vfs: std::ptr::null_mut(),
-            builtin_vfs_count: 0,
+            register_vtab_module,
+            #[cfg(feature = "fs")]
+            vfs_interface: limbo_ext::VfsInterface {
+                register_vfs: dynamic::register_vfs,
+                builtin_vfs: std::ptr::null_mut(),
+                builtin_vfs_count: 0,
+            },
         }
     }
 
@@ -290,31 +212,3 @@ impl Connection {
         Ok(())
     }
 }
-
-fn add_vfs_module(name: String, vfs: Arc<VfsMod>) {
-    let mut modules = VFS_MODULES
-        .get_or_init(|| Mutex::new(Vec::new()))
-        .lock()
-        .unwrap();
-    if !modules.iter().any(|v| v.0 == name) {
-        modules.push((name, vfs));
-    }
-}
-
-pub fn list_vfs_modules() -> Vec<String> {
-    VFS_MODULES
-        .get_or_init(|| Mutex::new(Vec::new()))
-        .lock()
-        .unwrap()
-        .iter()
-        .map(|v| v.0.clone())
-        .collect()
-}
-
-fn get_vfs_modules() -> Vec<Vfs> {
-    VFS_MODULES
-        .get_or_init(|| Mutex::new(Vec::new()))
-        .lock()
-        .unwrap()
-        .clone()
-}

core/function.rs

@@ -250,7 +250,7 @@ pub enum ScalarFunc {
     ZeroBlob,
     LastInsertRowid,
     Replace,
-    #[cfg(not(target_family = "wasm"))]
+    #[cfg(feature = "fs")]
     LoadExtension,
     StrfTime,
     Printf,
@@ -304,7 +304,7 @@ impl Display for ScalarFunc {
             Self::LastInsertRowid => "last_insert_rowid".to_string(),
             Self::Replace => "replace".to_string(),
             Self::DateTime => "datetime".to_string(),
-            #[cfg(not(target_family = "wasm"))]
+            #[cfg(feature = "fs")]
             Self::LoadExtension => "load_extension".to_string(),
             Self::StrfTime => "strftime".to_string(),
             Self::Printf => "printf".to_string(),
@@ -615,7 +615,7 @@ impl Func {
             "tan" => Ok(Self::Math(MathFunc::Tan)),
             "tanh" => Ok(Self::Math(MathFunc::Tanh)),
             "trunc" => Ok(Self::Math(MathFunc::Trunc)),
-            #[cfg(not(target_family = "wasm"))]
+            #[cfg(feature = "fs")]
             "load_extension" => Ok(Self::Scalar(ScalarFunc::LoadExtension)),
             "strftime" => Ok(Self::Scalar(ScalarFunc::StrfTime)),
             "printf" => Ok(Self::Scalar(ScalarFunc::Printf)),

core/io/mod.rs

@@ -213,6 +213,7 @@ cfg_block! {
 }
 
 mod memory;
+#[cfg(feature = "fs")]
 mod vfs;
 pub use memory::MemoryIO;
 mod common;

core/lib.rs

@@ -24,55 +24,48 @@ mod vector;
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-use ext::list_vfs_modules;
-use fallible_iterator::FallibleIterator;
-use fast_lock::SpinLock;
-use limbo_ext::{ResultCode, VTabKind, VTabModuleImpl};
-use limbo_sqlite3_parser::{ast, ast::Cmd, lexer::sql::Parser};
-use parking_lot::RwLock;
-use schema::{Column, Schema};
-use std::borrow::Cow;
-use std::cell::Cell;
-use std::collections::HashMap;
-use std::num::NonZero;
-use std::ops::Deref;
-use std::sync::{Arc, OnceLock};
-use std::{cell::RefCell, rc::Rc};
-use storage::btree::btree_init_page;
-#[cfg(feature = "fs")]
-use storage::database::FileStorage;
-use storage::page_cache::DumbLruPageCache;
-use storage::pager::allocate_page;
-pub use storage::pager::PageRef;
-use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE};
-pub use storage::wal::CheckpointMode;
-pub use storage::wal::WalFile;
-pub use storage::wal::WalFileShared;
-pub use types::OwnedValue;
-use util::{columns_from_create_table_body, parse_schema_rows};
-use vdbe::builder::QueryMode;
-use vdbe::VTabOpaqueCursor;
-
+use crate::{fast_lock::SpinLock, translate::optimizer::optimize_plan};
 pub use error::LimboError;
-use translate::select::prepare_select_plan;
-pub type Result<T, E = LimboError> = std::result::Result<T, E>;
-
-use crate::storage::wal::CheckpointResult;
-use crate::translate::optimizer::optimize_plan;
-pub use io::OpenFlags;
-pub use io::PlatformIO;
+use fallible_iterator::FallibleIterator;
 #[cfg(all(feature = "fs", target_family = "unix"))]
 pub use io::UnixIO;
 #[cfg(all(feature = "fs", target_os = "linux", feature = "io_uring"))]
 pub use io::UringIO;
-pub use io::{Buffer, Completion, File, MemoryIO, WriteCompletion, IO};
-pub use storage::buffer_pool::BufferPool;
-pub use storage::database::DatabaseStorage;
-pub use storage::pager::Page;
-pub use storage::pager::Pager;
-pub use storage::wal::CheckpointStatus;
-pub use storage::wal::Wal;
+pub use io::{Buffer, Completion, File, MemoryIO, OpenFlags, PlatformIO, WriteCompletion, IO};
+use limbo_ext::{ResultCode, VTabKind, VTabModuleImpl};
+use limbo_sqlite3_parser::{ast, ast::Cmd, lexer::sql::Parser};
+use parking_lot::RwLock;
+use schema::{Column, Schema};
+use std::{
+    borrow::Cow,
+    cell::{Cell, RefCell},
+    collections::HashMap,
+    num::NonZero,
+    ops::Deref,
+    rc::Rc,
+    sync::{Arc, OnceLock},
+};
+use storage::btree::btree_init_page;
+#[cfg(feature = "fs")]
+use storage::database::FileStorage;
+pub use storage::{
+    buffer_pool::BufferPool,
+    database::DatabaseStorage,
+    pager::PageRef,
+    pager::{Page, Pager},
+    wal::{CheckpointMode, CheckpointResult, CheckpointStatus, Wal, WalFile, WalFileShared},
+};
+use storage::{
+    page_cache::DumbLruPageCache,
+    pager::allocate_page,
+    sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE},
+};
+use translate::select::prepare_select_plan;
+pub use types::OwnedValue;
+use util::{columns_from_create_table_body, parse_schema_rows};
+use vdbe::{builder::QueryMode, VTabOpaqueCursor};
 
+pub type Result<T, E = LimboError> = std::result::Result<T, E>;
 pub static DATABASE_VERSION: OnceLock<String> = OnceLock::new();
 
 #[derive(Clone, PartialEq, Eq)]
@@ -209,7 +202,7 @@ impl Database {
     #[cfg(feature = "fs")]
     #[allow(clippy::arc_with_non_send_sync)]
     pub fn open_new(path: &str, vfs: &str) -> Result<(Arc<dyn IO>, Arc<Database>)> {
-        let vfsmods = ext::add_builtin_vfs_extensions(None)?;
+        let vfsmods = crate::ext::add_builtin_vfs_extensions(None)?;
         let io: Arc<dyn IO> = match vfsmods.iter().find(|v| v.0 == vfs).map(|v| v.1.clone()) {
             Some(vfs) => vfs,
             None => match vfs.trim() {
@@ -520,16 +513,16 @@ impl Connection {
         let mut all_vfs = vec![String::from("memory")];
         #[cfg(feature = "fs")]
         {
-            #[cfg(all(feature = "fs", target_family = "unix"))]
+            #[cfg(target_family = "unix")]
             {
                 all_vfs.push("syscall".to_string());
             }
-            #[cfg(all(feature = "fs", target_os = "linux", feature = "io_uring"))]
+            #[cfg(all(target_os = "linux", feature = "io_uring"))]
             {
                 all_vfs.push("io_uring".to_string());
             }
+            all_vfs.extend(crate::ext::list_vfs_modules());
         }
-        all_vfs.extend(list_vfs_modules());
         all_vfs
     }
 }

core/translate/expr.rs

@@ -1336,7 +1336,7 @@ pub fn translate_expr(
                             });
                             Ok(target_register)
                         }
-                        #[cfg(not(target_family = "wasm"))]
+                        #[cfg(feature = "fs")]
                         ScalarFunc::LoadExtension => {
                             let args = expect_arguments_exact!(args, 1, srf);
                             let start_reg = program.alloc_register();

core/vdbe/mod.rs

@@ -2639,7 +2639,7 @@ impl Program {
                                 let replacement = &state.registers[*start_reg + 2];
                                 state.registers[*dest] = exec_replace(source, pattern, replacement);
                             }
-                            #[cfg(not(target_family = "wasm"))]
+                            #[cfg(feature = "fs")]
                             ScalarFunc::LoadExtension => {
                                 let extension = &state.registers[*start_reg];
                                 let ext = resolve_ext_path(&extension.to_string())?;

extensions/core/Cargo.toml

@@ -8,6 +8,7 @@ repository.workspace = true
 description = "Limbo extensions core"
 
 [features]
+vfs = []
 core_only = []
 static = []
 

extensions/core/src/functions.rs

@@ -0,0 +1,38 @@
+use crate::{ResultCode, Value};
+use std::{
+    ffi::{c_char, c_void},
+    fmt::Display,
+};
+
+pub type ScalarFunction = unsafe extern "C" fn(argc: i32, *const Value) -> Value;
+
+pub type RegisterScalarFn =
+    unsafe extern "C" fn(ctx: *mut c_void, name: *const c_char, func: ScalarFunction) -> ResultCode;
+
+pub type RegisterAggFn = unsafe extern "C" fn(
+    ctx: *mut c_void,
+    name: *const c_char,
+    args: i32,
+    init: InitAggFunction,
+    step: StepFunction,
+    finalize: FinalizeFunction,
+) -> ResultCode;
+
+pub type InitAggFunction = unsafe extern "C" fn() -> *mut AggCtx;
+pub type StepFunction = unsafe extern "C" fn(ctx: *mut AggCtx, argc: i32, argv: *const Value);
+pub type FinalizeFunction = unsafe extern "C" fn(ctx: *mut AggCtx) -> Value;
+
+#[repr(C)]
+pub struct AggCtx {
+    pub state: *mut c_void,
+}
+
+pub trait AggFunc {
+    type State: Default;
+    type Error: Display;
+    const NAME: &'static str;
+    const ARGS: i32;
+
+    fn step(state: &mut Self::State, args: &[Value]);
+    fn finalize(state: Self::State) -> Result<Value, Self::Error>;
+}

extensions/core/src/lib.rs

@@ -1,29 +1,36 @@
+mod functions;
 mod types;
+#[cfg(feature = "vfs")]
 mod vfs_modules;
-#[cfg(not(target_family = "wasm"))]
+mod vtabs;
+pub use functions::{
+    AggCtx, AggFunc, FinalizeFunction, InitAggFunction, ScalarFunction, StepFunction,
+};
+use functions::{RegisterAggFn, RegisterScalarFn};
+#[cfg(feature = "vfs")]
 pub use limbo_macros::VfsDerive;
 pub use limbo_macros::{register_extension, scalar, AggregateDerive, VTabModuleDerive};
-use std::{
-    fmt::Display,
-    os::raw::{c_char, c_void},
-};
+use std::os::raw::c_void;
 pub use types::{ResultCode, Value, ValueType};
-pub use vfs_modules::{RegisterVfsFn, VfsFileImpl, VfsImpl};
-#[cfg(not(target_family = "wasm"))]
-pub use vfs_modules::{VfsExtension, VfsFile};
+#[cfg(feature = "vfs")]
+pub use vfs_modules::{RegisterVfsFn, VfsExtension, VfsFile, VfsFileImpl, VfsImpl, VfsInterface};
+use vtabs::RegisterModuleFn;
+pub use vtabs::{VTabCursor, VTabKind, VTabModule, VTabModuleImpl};
 
 pub type ExtResult<T> = std::result::Result<T, ResultCode>;
 
+pub type ExtensionEntryPoint = unsafe extern "C" fn(api: *const ExtensionApi) -> ResultCode;
+
 #[repr(C)]
 pub struct ExtensionApi {
     pub ctx: *mut c_void,
     pub register_scalar_function: RegisterScalarFn,
     pub register_aggregate_function: RegisterAggFn,
-    pub register_module: RegisterModuleFn,
-    pub register_vfs: RegisterVfsFn,
-    pub builtin_vfs: *mut *const VfsImpl,
-    pub builtin_vfs_count: i32,
+    pub register_vtab_module: RegisterModuleFn,
+    #[cfg(feature = "vfs")]
+    pub vfs_interface: VfsInterface,
 }
+
 unsafe impl Send for ExtensionApi {}
 unsafe impl Send for ExtensionApiRef {}
 
@@ -31,155 +38,3 @@ unsafe impl Send for ExtensionApiRef {}
 pub struct ExtensionApiRef {
     pub api: *const ExtensionApi,
 }
-
-impl ExtensionApi {
-    /// Since we want the option to build in extensions at compile time as well,
-    /// we add a slice of VfsImpls to the extension API, and this is called with any
-    /// libraries that we load staticly that will add their VFS implementations to the list.
-    pub fn add_builtin_vfs(&mut self, vfs: *const VfsImpl) -> ResultCode {
-        if vfs.is_null() || self.builtin_vfs.is_null() {
-            return ResultCode::Error;
-        }
-        let mut new = unsafe {
-            let slice =
-                std::slice::from_raw_parts_mut(self.builtin_vfs, self.builtin_vfs_count as usize);
-            Vec::from(slice)
-        };
-        new.push(vfs);
-        self.builtin_vfs = Box::into_raw(new.into_boxed_slice()) as *mut *const VfsImpl;
-        self.builtin_vfs_count += 1;
-        ResultCode::OK
-    }
-}
-
-pub type ExtensionEntryPoint = unsafe extern "C" fn(api: *const ExtensionApi) -> ResultCode;
-
-pub type ScalarFunction = unsafe extern "C" fn(argc: i32, *const Value) -> Value;
-
-pub type RegisterScalarFn =
-    unsafe extern "C" fn(ctx: *mut c_void, name: *const c_char, func: ScalarFunction) -> ResultCode;
-
-pub type RegisterAggFn = unsafe extern "C" fn(
-    ctx: *mut c_void,
-    name: *const c_char,
-    args: i32,
-    init: InitAggFunction,
-    step: StepFunction,
-    finalize: FinalizeFunction,
-) -> ResultCode;
-
-pub type RegisterModuleFn = unsafe extern "C" fn(
-    ctx: *mut c_void,
-    name: *const c_char,
-    module: VTabModuleImpl,
-    kind: VTabKind,
-) -> ResultCode;
-
-pub type InitAggFunction = unsafe extern "C" fn() -> *mut AggCtx;
-pub type StepFunction = unsafe extern "C" fn(ctx: *mut AggCtx, argc: i32, argv: *const Value);
-pub type FinalizeFunction = unsafe extern "C" fn(ctx: *mut AggCtx) -> Value;
-
-#[repr(C)]
-pub struct AggCtx {
-    pub state: *mut c_void,
-}
-
-pub trait AggFunc {
-    type State: Default;
-    type Error: Display;
-    const NAME: &'static str;
-    const ARGS: i32;
-
-    fn step(state: &mut Self::State, args: &[Value]);
-    fn finalize(state: Self::State) -> Result<Value, Self::Error>;
-}
-
-#[repr(C)]
-#[derive(Clone, Debug)]
-pub struct VTabModuleImpl {
-    pub ctx: *const c_void,
-    pub name: *const c_char,
-    pub create_schema: VtabFnCreateSchema,
-    pub open: VtabFnOpen,
-    pub filter: VtabFnFilter,
-    pub column: VtabFnColumn,
-    pub next: VtabFnNext,
-    pub eof: VtabFnEof,
-    pub update: VtabFnUpdate,
-    pub rowid: VtabRowIDFn,
-}
-
-#[cfg(feature = "core_only")]
-impl VTabModuleImpl {
-    pub fn init_schema(&self, args: Vec<Value>) -> ExtResult<String> {
-        let schema = unsafe { (self.create_schema)(args.as_ptr(), args.len() as i32) };
-        if schema.is_null() {
-            return Err(ResultCode::InvalidArgs);
-        }
-        for arg in args {
-            unsafe { arg.__free_internal_type() };
-        }
-        let schema = unsafe { std::ffi::CString::from_raw(schema) };
-        Ok(schema.to_string_lossy().to_string())
-    }
-}
-
-pub type VtabFnCreateSchema = unsafe extern "C" fn(args: *const Value, argc: i32) -> *mut c_char;
-
-pub type VtabFnOpen = unsafe extern "C" fn(*const c_void) -> *const c_void;
-
-pub type VtabFnFilter =
-    unsafe extern "C" fn(cursor: *const c_void, argc: i32, argv: *const Value) -> ResultCode;
-
-pub type VtabFnColumn = unsafe extern "C" fn(cursor: *const c_void, idx: u32) -> Value;
-
-pub type VtabFnNext = unsafe extern "C" fn(cursor: *const c_void) -> ResultCode;
-
-pub type VtabFnEof = unsafe extern "C" fn(cursor: *const c_void) -> bool;
-
-pub type VtabRowIDFn = unsafe extern "C" fn(cursor: *const c_void) -> i64;
-
-pub type VtabFnUpdate = unsafe extern "C" fn(
-    vtab: *const c_void,
-    argc: i32,
-    argv: *const Value,
-    p_out_rowid: *mut i64,
-) -> ResultCode;
-
-#[repr(C)]
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum VTabKind {
-    VirtualTable,
-    TableValuedFunction,
-}
-
-pub trait VTabModule: 'static {
-    type VCursor: VTabCursor<Error = Self::Error>;
-    const VTAB_KIND: VTabKind;
-    const NAME: &'static str;
-    type Error: std::fmt::Display;
-
-    fn create_schema(args: &[Value]) -> String;
-    fn open(&self) -> Result<Self::VCursor, Self::Error>;
-    fn filter(cursor: &mut Self::VCursor, args: &[Value]) -> ResultCode;
-    fn column(cursor: &Self::VCursor, idx: u32) -> Result<Value, Self::Error>;
-    fn next(cursor: &mut Self::VCursor) -> ResultCode;
-    fn eof(cursor: &Self::VCursor) -> bool;
-    fn update(&mut self, _rowid: i64, _args: &[Value]) -> Result<(), Self::Error> {
-        Ok(())
-    }
-    fn insert(&mut self, _args: &[Value]) -> Result<i64, Self::Error> {
-        Ok(0)
-    }
-    fn delete(&mut self, _rowid: i64) -> Result<(), Self::Error> {
-        Ok(())
-    }
-}
-
-pub trait VTabCursor: Sized {
-    type Error: std::fmt::Display;
-    fn rowid(&self) -> i64;
-    fn column(&self, idx: u32) -> Result<Value, Self::Error>;
-    fn eof(&self) -> bool;
-    fn next(&mut self) -> ResultCode;
-}

extensions/core/src/vfs_modules.rs

@@ -1,7 +1,16 @@
-use crate::{ExtResult, ResultCode};
+use crate::{ExtResult, ExtensionApi, ResultCode};
 use std::ffi::{c_char, c_void};
 
-#[cfg(not(target_family = "wasm"))]
+/// Field for ExtensionApi to interface with VFS extensions,
+/// separated to more easily feature flag out for WASM builds.
+#[repr(C)]
+pub struct VfsInterface {
+    pub register_vfs: RegisterVfsFn,
+    pub builtin_vfs: *mut *const VfsImpl,
+    pub builtin_vfs_count: i32,
+}
+unsafe impl Send for VfsInterface {}
+
 pub trait VfsExtension: Default + Send + Sync {
     const NAME: &'static str;
     type File: VfsFile;
@@ -21,7 +30,7 @@ pub trait VfsExtension: Default + Send + Sync {
         chrono::Local::now().format("%Y-%m-%d %H:%M:%S").to_string()
     }
 }
-#[cfg(not(target_family = "wasm"))]
+
 pub trait VfsFile: Send + Sync {
     fn lock(&mut self, _exclusive: bool) -> ExtResult<()> {
         Ok(())
@@ -112,3 +121,26 @@ impl Drop for VfsFileImpl {
         }
     }
 }
+
+impl ExtensionApi {
+    /// Since we want the option to build in extensions at compile time as well,
+    /// we add a slice of VfsImpls to the extension API, and this is called with any
+    /// libraries that we load staticly that will add their VFS implementations to the list.
+    pub fn add_builtin_vfs(&mut self, vfs: *const VfsImpl) -> ResultCode {
+        if vfs.is_null() || self.vfs_interface.builtin_vfs.is_null() {
+            return ResultCode::Error;
+        }
+        let mut new = unsafe {
+            let slice = std::slice::from_raw_parts_mut(
+                self.vfs_interface.builtin_vfs,
+                self.vfs_interface.builtin_vfs_count as usize,
+            );
+            Vec::from(slice)
+        };
+        new.push(vfs);
+        self.vfs_interface.builtin_vfs =
+            Box::into_raw(new.into_boxed_slice()) as *mut *const VfsImpl;
+        self.vfs_interface.builtin_vfs_count += 1;
+        ResultCode::OK
+    }
+}

extensions/core/src/vtabs.rs

@@ -0,0 +1,99 @@
+use crate::{ExtResult, ResultCode, Value};
+use std::ffi::{c_char, c_void};
+
+pub type RegisterModuleFn = unsafe extern "C" fn(
+    ctx: *mut c_void,
+    name: *const c_char,
+    module: VTabModuleImpl,
+    kind: VTabKind,
+) -> ResultCode;
+
+#[repr(C)]
+#[derive(Clone, Debug)]
+pub struct VTabModuleImpl {
+    pub ctx: *const c_void,
+    pub name: *const c_char,
+    pub create_schema: VtabFnCreateSchema,
+    pub open: VtabFnOpen,
+    pub filter: VtabFnFilter,
+    pub column: VtabFnColumn,
+    pub next: VtabFnNext,
+    pub eof: VtabFnEof,
+    pub update: VtabFnUpdate,
+    pub rowid: VtabRowIDFn,
+}
+
+#[cfg(feature = "core_only")]
+impl VTabModuleImpl {
+    pub fn init_schema(&self, args: Vec<Value>) -> ExtResult<String> {
+        let schema = unsafe { (self.create_schema)(args.as_ptr(), args.len() as i32) };
+        if schema.is_null() {
+            return Err(ResultCode::InvalidArgs);
+        }
+        for arg in args {
+            unsafe { arg.__free_internal_type() };
+        }
+        let schema = unsafe { std::ffi::CString::from_raw(schema) };
+        Ok(schema.to_string_lossy().to_string())
+    }
+}
+
+pub type VtabFnCreateSchema = unsafe extern "C" fn(args: *const Value, argc: i32) -> *mut c_char;
+
+pub type VtabFnOpen = unsafe extern "C" fn(*const c_void) -> *const c_void;
+
+pub type VtabFnFilter =
+    unsafe extern "C" fn(cursor: *const c_void, argc: i32, argv: *const Value) -> ResultCode;
+
+pub type VtabFnColumn = unsafe extern "C" fn(cursor: *const c_void, idx: u32) -> Value;
+
+pub type VtabFnNext = unsafe extern "C" fn(cursor: *const c_void) -> ResultCode;
+
+pub type VtabFnEof = unsafe extern "C" fn(cursor: *const c_void) -> bool;
+
+pub type VtabRowIDFn = unsafe extern "C" fn(cursor: *const c_void) -> i64;
+
+pub type VtabFnUpdate = unsafe extern "C" fn(
+    vtab: *const c_void,
+    argc: i32,
+    argv: *const Value,
+    p_out_rowid: *mut i64,
+) -> ResultCode;
+
+#[repr(C)]
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum VTabKind {
+    VirtualTable,
+    TableValuedFunction,
+}
+
+pub trait VTabModule: 'static {
+    type VCursor: VTabCursor<Error = Self::Error>;
+    const VTAB_KIND: VTabKind;
+    const NAME: &'static str;
+    type Error: std::fmt::Display;
+
+    fn create_schema(args: &[Value]) -> String;
+    fn open(&self) -> Result<Self::VCursor, Self::Error>;
+    fn filter(cursor: &mut Self::VCursor, args: &[Value]) -> ResultCode;
+    fn column(cursor: &Self::VCursor, idx: u32) -> Result<Value, Self::Error>;
+    fn next(cursor: &mut Self::VCursor) -> ResultCode;
+    fn eof(cursor: &Self::VCursor) -> bool;
+    fn update(&mut self, _rowid: i64, _args: &[Value]) -> Result<(), Self::Error> {
+        Ok(())
+    }
+    fn insert(&mut self, _args: &[Value]) -> Result<i64, Self::Error> {
+        Ok(0)
+    }
+    fn delete(&mut self, _rowid: i64) -> Result<(), Self::Error> {
+        Ok(())
+    }
+}
+
+pub trait VTabCursor: Sized {
+    type Error: std::fmt::Display;
+    fn rowid(&self) -> i64;
+    fn column(&self, idx: u32) -> Result<Value, Self::Error>;
+    fn eof(&self) -> bool;
+    fn next(&mut self) -> ResultCode;
+}

extensions/tests/Cargo.toml

@@ -15,7 +15,7 @@ static= [ "limbo_ext/static" ]
 [dependencies]
 env_logger = "0.11.6"
 lazy_static = "1.5.0"
-limbo_ext = { workspace = true, features = ["static"] }
+limbo_ext = { workspace = true, features = ["static", "vfs"] }
 log = "0.4.26"
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]

macros/src/lib.rs

@@ -615,7 +615,7 @@ pub fn derive_vtab_module(input: TokenStream) -> TokenStream {
                     update: Self::#update_fn_name,
                     rowid: Self::#rowid_fn_name,
                 };
-                (api.register_module)(api.ctx, name_c, module, <#struct_name as ::limbo_ext::VTabModule>::VTAB_KIND)
+                (api.register_vtab_module)(api.ctx, name_c, module, <#struct_name as ::limbo_ext::VTabModule>::VTAB_KIND)
             }
         }
     };
@@ -686,7 +686,7 @@ pub fn derive_vfs_module(input: TokenStream) -> TokenStream {
                 current_time: #get_current_time_fn_name,
             };
             let vfsimpl = ::std::boxed::Box::into_raw(::std::boxed::Box::new(vfs_mod)) as *const ::limbo_ext::VfsImpl;
-            (api.register_vfs)(name, vfsimpl)
+            (api.vfs_interface.register_vfs)(name, vfsimpl)
         }
 
         #[no_mangle]