mirrors/limbo
1
0
Fork 0
mirror of https://github.com/tursodatabase/limbo.git synced 2025-08-04 01:58:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge branch 'main' into delete-btree-row

Browse source
This commit is contained in:
Krishna Vishal 2025-02-07 01:15:21 +05:30 • committed by GitHub
commit 7cb7eb4e65
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
44 changed files with 2144 additions and 382 deletions
Download patch file Download diff file Expand all files Collapse all files

94
Cargo.lock generated
View file

@ -140,6 +140,12 @@ version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
[[package]]
name = "arrayref"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
[[package]]
name = "arrayvec"
version = "0.7.6"
@ -204,6 +210,19 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36"
[[package]]
name = "blake3"
version = "1.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
]
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -455,6 +474,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "constant_time_eq"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
@ -617,6 +642,12 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "data-encoding"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f"
[[package]]
name = "debugid"
version = "0.8.0"
@ -1580,10 +1611,12 @@ dependencies = [
"julian_day_converter",
"libc",
"libloading",
"limbo_crypto",
"limbo_ext",
"limbo_macros",
"limbo_percentile",
"limbo_regexp",
"limbo_series",
"limbo_time",
"limbo_uuid",
"log",
@ -1612,6 +1645,19 @@ dependencies = [
"tracing",
]
[[package]]
name = "limbo_crypto"
version = "0.0.14"
dependencies = [
"blake3",
"data-encoding",
"limbo_ext",
"md5",
"mimalloc",
"ring",
"urlencoding",
]
[[package]]
name = "limbo_ext"
version = "0.0.14"
@ -1656,6 +1702,15 @@ dependencies = [
"regex",
]
[[package]]
name = "limbo_series"
version = "0.0.14"
dependencies = [
"limbo_ext",
"log",
"mimalloc",
]
[[package]]
name = "limbo_sim"
version = "0.0.14"
@ -1732,6 +1787,12 @@ version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f"
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.7.4"
@ -2517,6 +2578,21 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "ring"
version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
dependencies = [
"cc",
"cfg-if",
"getrandom 0.2.15",
"libc",
"spin",
"untrusted",
"windows-sys 0.52.0",
]
[[package]]
name = "rstest"
version = "0.18.2"
@ -2740,6 +2816,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
[[package]]
name = "sqlite3-parser"
version = "0.13.0"
@ -3109,6 +3191,12 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
[[package]]
name = "untrusted"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "url"
version = "2.5.4"
@ -3120,6 +3208,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf16_iter"
version = "1.0.5"

2
Cargo.toml
View file

@ -19,6 +19,8 @@ members = [
"tests",
"extensions/percentile",
"extensions/time",
"extensions/crypto",
"extensions/series",
]
exclude = ["perf/latency/limbo"]

8
bindings/java/example/build.gradle.kts
View file

@ -2,7 +2,7 @@ plugins {
id("java")
}
group = "org.github.seonwkim"
group = "org.github.tursodatabase"
version = "1.0-SNAPSHOT"
repositories {
@ -19,3 +19,9 @@ dependencies {
tasks.test {
useJUnitPlatform()
}
tasks.register<JavaExec>("run") {
group = "application"
classpath = sourceSets["main"].runtimeClasspath
mainClass.set("org.github.tursodatabase.Main")
}

8
bindings/java/example/src/main/java/org/github/seonwkim/Main.java → bindings/java/example/src/main/java/org/github/tursodatabase/Main.java
View file

@ -1,4 +1,4 @@
package org.github.seonwkim;
package org.github.tursodatabase;
import java.sql.Connection;
import java.sql.DriverManager;
@ -14,9 +14,9 @@ public class Main {
ResultSet.CONCUR_READ_ONLY,
ResultSet.CLOSE_CURSORS_AT_COMMIT);
stmt.execute("CREATE TABLE users (id INT PRIMARY KEY, username TEXT);");
stmt.execute("INSERT INTO users VALUES (1, 'seonwoo');");
stmt.execute("INSERT INTO users VALUES (2, 'seonwoo');");
stmt.execute("INSERT INTO users VALUES (3, 'seonwoo');");
stmt.execute("INSERT INTO users VALUES (1, 'limbo');");
stmt.execute("INSERT INTO users VALUES (2, 'turso');");
stmt.execute("INSERT INTO users VALUES (3, 'who knows');");
stmt.execute("SELECT * FROM users");
System.out.println(
"result: " + stmt.getResultSet().getInt(1) + ", " + stmt.getResultSet().getString(2));

4
core/Cargo.toml
View file

@ -26,6 +26,8 @@ io_uring = ["dep:io-uring", "rustix/io_uring"]
percentile = ["limbo_percentile/static"]
regexp = ["limbo_regexp/static"]
time = ["limbo_time/static"]
crypto = ["limbo_crypto/static"]
series = ["limbo_series/static"]
[target.'cfg(target_os = "linux")'.dependencies]
io-uring = { version = "0.6.1", optional = true }
@ -65,6 +67,8 @@ limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static
limbo_regexp = { path = "../extensions/regexp", optional = true, features = ["static"] }
limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] }
limbo_time = { path = "../extensions/time", optional = true, features = ["static"] }
limbo_crypto = { path = "../extensions/crypto", optional = true, features = ["static"] }
limbo_series = { path = "../extensions/series", optional = true, features = ["static"] }
miette = "7.4.0"
strum = "0.26"
parking_lot = "0.12.3"

90
core/ext/mod.rs
View file

@ -1,6 +1,11 @@
use crate::{function::ExternalFunc, Database};
use limbo_ext::{ExtensionApi, InitAggFunction, ResultCode, ScalarFunction};
use crate::{function::ExternalFunc, util::columns_from_create_table_body, Database, VirtualTable};
use fallible_iterator::FallibleIterator;
use limbo_ext::{ExtensionApi, InitAggFunction, ResultCode, ScalarFunction, VTabModuleImpl};
pub use limbo_ext::{FinalizeFunction, StepFunction, Value as ExtValue, ValueType as ExtValueType};
use sqlite3_parser::{
ast::{Cmd, Stmt},
lexer::sql::Parser,
};
use std::{
ffi::{c_char, c_void, CStr},
rc::Rc,
@ -44,6 +49,48 @@ unsafe extern "C" fn register_aggregate_function(
db.register_aggregate_function_impl(&name_str, args, (init_func, step_func, finalize_func))
}
unsafe extern "C" fn register_module(
ctx: *mut c_void,
name: *const c_char,
module: VTabModuleImpl,
) -> ResultCode {
let c_str = unsafe { CStr::from_ptr(name) };
let name_str = match c_str.to_str() {
Ok(s) => s.to_string(),
Err(_) => return ResultCode::Error,
};
if ctx.is_null() {
return ResultCode::Error;
}
let db = unsafe { &mut *(ctx as *mut Database) };
db.register_module_impl(&name_str, module)
}
unsafe extern "C" fn declare_vtab(
ctx: *mut c_void,
name: *const c_char,
sql: *const c_char,
) -> ResultCode {
let c_str = unsafe { CStr::from_ptr(name) };
let name_str = match c_str.to_str() {
Ok(s) => s.to_string(),
Err(_) => return ResultCode::Error,
};
let c_str = unsafe { CStr::from_ptr(sql) };
let sql_str = match c_str.to_str() {
Ok(s) => s.to_string(),
Err(_) => return ResultCode::Error,
};
if ctx.is_null() {
return ResultCode::Error;
}
let db = unsafe { &mut *(ctx as *mut Database) };
db.declare_vtab_impl(&name_str, &sql_str)
}
impl Database {
fn register_scalar_function_impl(&self, name: &str, func: ScalarFunction) -> ResultCode {
self.syms.borrow_mut().functions.insert(
@ -66,11 +113,42 @@ impl Database {
ResultCode::OK
}
fn register_module_impl(&mut self, name: &str, module: VTabModuleImpl) -> ResultCode {
self.vtab_modules.insert(name.to_string(), Rc::new(module));
ResultCode::OK
}
fn declare_vtab_impl(&mut self, name: &str, sql: &str) -> ResultCode {
let mut parser = Parser::new(sql.as_bytes());
let cmd = parser.next().unwrap().unwrap();
let Cmd::Stmt(stmt) = cmd else {
return ResultCode::Error;
};
let Stmt::CreateTable { body, .. } = stmt else {
return ResultCode::Error;
};
let Ok(columns) = columns_from_create_table_body(body) else {
return ResultCode::Error;
};
let vtab_module = self.vtab_modules.get(name).unwrap().clone();
let vtab = VirtualTable {
name: name.to_string(),
implementation: vtab_module,
columns,
args: None,
};
self.syms.borrow_mut().vtabs.insert(name.to_string(), vtab);
ResultCode::OK
}
pub fn build_limbo_ext(&self) -> ExtensionApi {
ExtensionApi {
ctx: self as *const _ as *mut c_void,
register_scalar_function,
register_aggregate_function,
register_module,
declare_vtab,
}
}
@ -92,6 +170,14 @@ impl Database {
if unsafe { !limbo_time::register_extension_static(&ext_api).is_ok() } {
return Err("Failed to register time extension".to_string());
}
#[cfg(feature = "crypto")]
if unsafe { !limbo_crypto::register_extension_static(&ext_api).is_ok() } {
return Err("Failed to register crypto extension".to_string());
}
#[cfg(feature = "series")]
if unsafe { !limbo_series::register_extension_static(&ext_api).is_ok() } {
return Err("Failed to register series extension".to_string());
}
Ok(())
}
}

10
core/vdbe/datetime.rs → core/functions/datetime.rs
View file

@ -29,7 +29,7 @@ pub fn exec_strftime(values: &[OwnedValue]) -> OwnedValue {
}
let format_str = match &values[0] {
OwnedValue::Text(text) => text.value.to_string(),
OwnedValue::Text(text) => text.as_str().to_string(),
OwnedValue::Integer(num) => num.to_string(),
OwnedValue::Float(num) => format!("{:.14}", num),
_ => return OwnedValue::Null,
@ -82,7 +82,7 @@ fn modify_dt(
if let OwnedValue::Text(ref text_rc) = modifier {
// TODO: to prevent double conversion and properly support 'utc'/'localtime', we also
// need to keep track of the current timezone and apply it to the modifier.
match apply_modifier(dt, &text_rc.value) {
match apply_modifier(dt, text_rc.as_str()) {
Ok(true) => subsec_requested = true,
Ok(false) => {}
Err(_) => return OwnedValue::build_text(Rc::new(String::new())),
@ -122,7 +122,7 @@ fn format_dt(dt: NaiveDateTime, output_type: DateTimeOutput, subsec: bool) -> St
// Not as fast as if the formatting was native to chrono, but a good enough
// for now, just to have the feature implemented
fn strftime_format(dt: &NaiveDateTime, format_str: &str) -> String {
use super::strftime::CustomStrftimeItems;
use crate::functions::strftime::CustomStrftimeItems;
use std::fmt::Write;
// Necessary to remove %f and %J that are exclusive formatters to sqlite
// Chrono does not support them, so it is necessary to replace the modifiers manually
@ -382,7 +382,7 @@ fn get_unixepoch_from_naive_datetime(value: NaiveDateTime) -> String {
fn parse_naive_date_time(time_value: &OwnedValue) -> Option<NaiveDateTime> {
match time_value {
OwnedValue::Text(s) => get_date_time_from_time_value_string(&s.value),
OwnedValue::Text(s) => get_date_time_from_time_value_string(s.as_str()),
OwnedValue::Integer(i) => get_date_time_from_time_value_integer(*i),
OwnedValue::Float(f) => get_date_time_from_time_value_float(*f),
_ => None,
@ -1100,7 +1100,7 @@ mod tests {
for (input, expected) in test_cases {
let result = exec_time(&[input]);
if let OwnedValue::Text(result_str) = result {
assert_eq!(result_str.value.as_str(), expected);
assert_eq!(result_str.as_str(), expected);
} else {
panic!("Expected OwnedValue::Text, but got: {:?}", result);
}

3
core/functions/mod.rs Normal file
View file

@ -0,0 +1,3 @@
pub mod datetime;
pub mod printf;
pub mod strftime;

4
core/vdbe/printf.rs → core/functions/printf.rs
View file

@ -9,7 +9,7 @@ pub fn exec_printf(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
return Ok(OwnedValue::Null);
}
let format_str = match &values[0] {
OwnedValue::Text(t) => &t.value,
OwnedValue::Text(t) => t.as_str(),
_ => return Ok(OwnedValue::Null),
};
@ -44,7 +44,7 @@ pub fn exec_printf(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
return Err(LimboError::InvalidArgument("not enough arguments".into()));
}
match &values[args_index] {
OwnedValue::Text(t) => result.push_str(&t.value),
OwnedValue::Text(t) => result.push_str(t.as_str()),
OwnedValue::Null => result.push_str("(null)"),
v => result.push_str(&v.to_string()),
}

0
core/vdbe/strftime.rs → core/functions/strftime.rs
View file

12
core/json/json_operations.rs
View file

@ -164,7 +164,7 @@ pub fn json_remove(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
.iter()
.map(|path| {
if let OwnedValue::Text(path) = path {
json_path(&path.value)
json_path(path.as_str())
} else {
crate::bail_constraint_error!("bad JSON path: {:?}", path.to_string())
}
@ -514,7 +514,7 @@ mod tests {
let result = json_remove(&args).unwrap();
match result {
OwnedValue::Text(t) => assert_eq!(t.value.as_str(), "[1,2,4,5]"),
OwnedValue::Text(t) => assert_eq!(t.as_str(), "[1,2,4,5]"),
_ => panic!("Expected Text value"),
}
}
@ -529,7 +529,7 @@ mod tests {
let result = json_remove(&args).unwrap();
match result {
OwnedValue::Text(t) => assert_eq!(t.value.as_str(), r#"{"b":2}"#),
OwnedValue::Text(t) => assert_eq!(t.as_str(), r#"{"b":2}"#),
_ => panic!("Expected Text value"),
}
}
@ -543,7 +543,7 @@ mod tests {
let result = json_remove(&args).unwrap();
match result {
OwnedValue::Text(t) => assert_eq!(t.value.as_str(), r#"{"a":{"b":{"d":2}}}"#),
OwnedValue::Text(t) => assert_eq!(t.as_str(), r#"{"a":{"b":{"d":2}}}"#),
_ => panic!("Expected Text value"),
}
}
@ -557,7 +557,7 @@ mod tests {
let result = json_remove(&args).unwrap();
match result {
OwnedValue::Text(t) => assert_eq!(t.value.as_str(), r#"{"a":2,"a":3}"#),
OwnedValue::Text(t) => assert_eq!(t.as_str(), r#"{"a":2,"a":3}"#),
_ => panic!("Expected Text value"),
}
}
@ -584,7 +584,7 @@ mod tests {
let result = json_remove(&args).unwrap();
match result {
OwnedValue::Text(t) => {
let value = t.value.as_str();
let value = t.as_str();
assert!(value.contains(r#"[1,3]"#));
assert!(value.contains(r#"{"x":2}"#));
}

93
core/json/mod.rs
View file

@ -73,7 +73,7 @@ pub fn get_json(json_value: &OwnedValue, indent: Option<&str>) -> crate::Result<
fn get_json_value(json_value: &OwnedValue) -> crate::Result<Val> {
match json_value {
OwnedValue::Text(ref t) => match from_str::<Val>(&t.value) {
OwnedValue::Text(ref t) => match from_str::<Val>(t.as_str()) {
Ok(json) => Ok(json),
Err(_) => {
crate::bail_parse_error!("malformed JSON")
@ -104,9 +104,9 @@ pub fn json_array(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
OwnedValue::Blob(_) => crate::bail_constraint_error!("JSON cannot hold BLOB values"),
OwnedValue::Text(t) => {
if t.subtype == TextSubtype::Json {
s.push_str(&t.value);
s.push_str(t.as_str());
} else {
match to_string(&*t.value) {
match to_string(&t.as_str().to_string()) {
Ok(json) => s.push_str(&json),
Err(_) => crate::bail_parse_error!("malformed JSON"),
}
@ -166,10 +166,12 @@ pub fn json_set(json: &OwnedValue, values: &[OwnedValue]) -> crate::Result<Owned
if let Some(path) = path {
let new_value = match value {
OwnedValue::Text(Text {
value,
subtype: TextSubtype::Text,
}) => Val::String(value.to_string()),
OwnedValue::Text(
t @ Text {
subtype: TextSubtype::Text,
..
},
) => Val::String(t.as_str().to_string()),
_ => get_json_value(value)?,
};
@ -323,7 +325,7 @@ fn convert_db_type_to_json(value: &OwnedValue) -> crate::Result<Val> {
OwnedValue::Text(t) => match t.subtype {
// Convert only to json if the subtype is json (if we got it from another json function)
TextSubtype::Json => get_json_value(value)?,
TextSubtype::Text => Val::String(t.value.to_string()),
TextSubtype::Text => Val::String(t.as_str().to_string()),
},
OwnedValue::Blob(_) => crate::bail_constraint_error!("JSON cannot hold BLOB values"),
unsupported_value => crate::bail_constraint_error!(
@ -431,18 +433,21 @@ fn json_extract_single<'a>(
fn json_path_from_owned_value(path: &OwnedValue, strict: bool) -> crate::Result<Option<JsonPath>> {
let json_path = if strict {
match path {
OwnedValue::Text(t) => json_path(t.value.as_str())?,
OwnedValue::Text(t) => json_path(t.as_str())?,
OwnedValue::Null => return Ok(None),
_ => crate::bail_constraint_error!("JSON path error near: {:?}", path.to_string()),
}
} else {
match path {
OwnedValue::Text(t) => {
if t.value.starts_with("$") {
json_path(t.value.as_str())?
if t.as_str().starts_with("$") {
json_path(t.as_str())?
} else {
JsonPath {
elements: vec![PathElement::Root(), PathElement::Key(t.value.to_string())],
elements: vec![
PathElement::Root(),
PathElement::Key(t.as_str().to_string()),
],
}
}
}
@ -606,7 +611,7 @@ fn find_or_create_target<'a>(json: &'a mut Val, path: &JsonPath) -> Option<Targe
pub fn json_error_position(json: &OwnedValue) -> crate::Result<OwnedValue> {
match json {
OwnedValue::Text(t) => match from_str::<Val>(&t.value) {
OwnedValue::Text(t) => match from_str::<Val>(t.as_str()) {
Ok(_) => Ok(OwnedValue::Integer(0)),
Err(JsonError::Message { location, .. }) => {
if let Some(loc) = location {
@ -639,7 +644,7 @@ pub fn json_object(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
.map(|chunk| match chunk {
[key, value] => {
let key = match key {
OwnedValue::Text(t) => t.value.to_string(),
OwnedValue::Text(t) => t.as_str().to_string(),
_ => crate::bail_constraint_error!("labels must be TEXT"),
};
let json_val = convert_db_type_to_json(value)?;
@ -656,7 +661,7 @@ pub fn json_object(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
pub fn is_json_valid(json_value: &OwnedValue) -> crate::Result<OwnedValue> {
match json_value {
OwnedValue::Text(ref t) => match from_str::<Val>(&t.value) {
OwnedValue::Text(ref t) => match from_str::<Val>(t.as_str()) {
Ok(_) => Ok(OwnedValue::Integer(1)),
Err(_) => Ok(OwnedValue::Integer(0)),
},
@ -679,7 +684,7 @@ mod tests {
let input = OwnedValue::build_text(Rc::new("{ key: 'value' }".to_string()));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"key\":\"value\""));
assert!(result_str.as_str().contains("\"key\":\"value\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -691,7 +696,7 @@ mod tests {
let input = OwnedValue::build_text(Rc::new("{ key: ''value'' }".to_string()));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"key\":\"value\""));
assert!(result_str.as_str().contains("\"key\":\"value\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -703,7 +708,7 @@ mod tests {
let input = OwnedValue::build_text(Rc::new("{ \"key\": Infinity }".to_string()));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("{\"key\":9e999}"));
assert!(result_str.as_str().contains("{\"key\":9e999}"));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -715,7 +720,7 @@ mod tests {
let input = OwnedValue::build_text(Rc::new("{ \"key\": -Infinity }".to_string()));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("{\"key\":-9e999}"));
assert!(result_str.as_str().contains("{\"key\":-9e999}"));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -727,7 +732,7 @@ mod tests {
let input = OwnedValue::build_text(Rc::new("{ \"key\": NaN }".to_string()));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("{\"key\":null}"));
assert!(result_str.as_str().contains("{\"key\":null}"));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -749,7 +754,7 @@ mod tests {
let input = OwnedValue::build_text(Rc::new("{\"key\":\"value\"}".to_string()));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"key\":\"value\""));
assert!(result_str.as_str().contains("\"key\":\"value\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -772,7 +777,7 @@ mod tests {
let input = OwnedValue::Blob(Rc::new(binary_json));
let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"asd\":\"adf\""));
assert!(result_str.as_str().contains("\"asd\":\"adf\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -809,7 +814,7 @@ mod tests {
let result = json_array(&input).unwrap();
if let OwnedValue::Text(res) = result {
assert_eq!(res.value.as_str(), "[\"value1\",\"value2\",1,1.1]");
assert_eq!(res.as_str(), "[\"value1\",\"value2\",1,1.1]");
assert_eq!(res.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -822,7 +827,7 @@ mod tests {
let result = json_array(&input).unwrap();
if let OwnedValue::Text(res) = result {
assert_eq!(res.value.as_str(), "[]");
assert_eq!(res.as_str(), "[]");
assert_eq!(res.subtype, TextSubtype::Json);
} else {
panic!("Expected OwnedValue::Text");
@ -1064,7 +1069,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(json_text.value.as_str(), r#"{"key":"value"}"#);
assert_eq!(json_text.as_str(), r#"{"key":"value"}"#);
}
#[test]
@ -1100,7 +1105,7 @@ mod tests {
panic!("Expected OwnedValue::Text");
};
assert_eq!(
json_text.value.as_str(),
json_text.as_str(),
r#"{"text_key":"text_value","json_key":{"json":"value","number":1},"integer_key":1,"float_key":1.1,"null_key":null}"#
);
}
@ -1115,7 +1120,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(json_text.value.as_str(), r#"{"key":{"json":"value"}}"#);
assert_eq!(json_text.as_str(), r#"{"key":{"json":"value"}}"#);
}
#[test]
@ -1128,10 +1133,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(
json_text.value.as_str(),
r#"{"key":"{\"json\":\"value\"}"}"#
);
assert_eq!(json_text.as_str(), r#"{"key":"{\"json\":\"value\"}"}"#);
}
#[test]
@ -1149,10 +1151,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(
json_text.value.as_str(),
r#"{"parent_key":{"key":"value"}}"#
);
assert_eq!(json_text.as_str(), r#"{"parent_key":{"key":"value"}}"#);
}
#[test]
@ -1165,7 +1164,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(json_text.value.as_str(), r#"{"key":"value"}"#);
assert_eq!(json_text.as_str(), r#"{"key":"value"}"#);
}
#[test]
@ -1176,7 +1175,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(json_text.value.as_str(), r#"{}"#);
assert_eq!(json_text.as_str(), r#"{}"#);
}
#[test]
@ -1301,10 +1300,7 @@ mod tests {
#[test]
fn test_json_path_from_owned_value_root_strict() {
let path = OwnedValue::Text(Text {
value: Rc::new("$".to_string()),
subtype: TextSubtype::Text,
});
let path = OwnedValue::Text(Text::new(Rc::new("$".to_string())));
let result = json_path_from_owned_value(&path, true);
assert!(result.is_ok());
@ -1321,10 +1317,7 @@ mod tests {
#[test]
fn test_json_path_from_owned_value_root_non_strict() {
let path = OwnedValue::Text(Text {
value: Rc::new("$".to_string()),
subtype: TextSubtype::Text,
});
let path = OwnedValue::Text(Text::new(Rc::new("$".to_string())));
let result = json_path_from_owned_value(&path, false);
assert!(result.is_ok());
@ -1341,20 +1334,14 @@ mod tests {
#[test]
fn test_json_path_from_owned_value_named_strict() {
let path = OwnedValue::Text(Text {
value: Rc::new("field".to_string()),
subtype: TextSubtype::Text,
});
let path = OwnedValue::Text(Text::new(Rc::new("field".to_string())));
assert!(json_path_from_owned_value(&path, true).is_err());
}
#[test]
fn test_json_path_from_owned_value_named_non_strict() {
let path = OwnedValue::Text(Text {
value: Rc::new("field".to_string()),
subtype: TextSubtype::Text,
});
let path = OwnedValue::Text(Text::new(Rc::new("field".to_string())));
let result = json_path_from_owned_value(&path, false);
assert!(result.is_ok());

80
core/lib.rs
View file

@ -1,6 +1,7 @@
mod error;
mod ext;
mod function;
mod functions;
mod info;
mod io;
#[cfg(feature = "json")]
@ -24,12 +25,13 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
use fallible_iterator::FallibleIterator;
#[cfg(not(target_family = "wasm"))]
use libloading::{Library, Symbol};
#[cfg(not(target_family = "wasm"))]
use limbo_ext::{ExtensionApi, ExtensionEntryPoint};
use limbo_ext::{ResultCode, VTabModuleImpl, Value as ExtValue};
use log::trace;
use parking_lot::RwLock;
use schema::Schema;
use sqlite3_parser::ast;
use sqlite3_parser::{ast::Cmd, lexer::sql::Parser};
use schema::{Column, Schema};
use sqlite3_parser::{ast, ast::Cmd, lexer::sql::Parser};
use std::cell::Cell;
use std::collections::HashMap;
use std::num::NonZero;
@ -43,9 +45,11 @@ use storage::pager::allocate_page;
use storage::sqlite3_ondisk::{DatabaseHeader, DATABASE_HEADER_SIZE};
pub use storage::wal::WalFile;
pub use storage::wal::WalFileShared;
use types::OwnedValue;
pub use types::Value;
use util::parse_schema_rows;
use vdbe::builder::QueryMode;
use vdbe::VTabOpaqueCursor;
pub use error::LimboError;
use translate::select::prepare_select_plan;
@ -81,6 +85,7 @@ pub struct Database {
schema: Rc<RefCell<Schema>>,
header: Rc<RefCell<DatabaseHeader>>,
syms: Rc<RefCell<SymbolTable>>,
vtab_modules: HashMap<String, Rc<VTabModuleImpl>>,
// Shared structures of a Database are the parts that are common to multiple threads that might
// create DB connections.
_shared_page_cache: Arc<RwLock<DumbLruPageCache>>,
@ -143,6 +148,7 @@ impl Database {
_shared_page_cache: _shared_page_cache.clone(),
_shared_wal: shared_wal.clone(),
syms,
vtab_modules: HashMap::new(),
};
if let Err(e) = db.register_builtins() {
return Err(LimboError::ExtensionError(e));
@ -266,8 +272,8 @@ impl Connection {
let sql = sql.as_ref();
trace!("Preparing: {}", sql);
let db = &self.db;
let syms: &SymbolTable = &db.syms.borrow();
let mut parser = Parser::new(sql.as_bytes());
let syms = &db.syms.borrow();
let cmd = parser.next()?;
if let Some(cmd) = cmd {
match cmd {
@ -411,7 +417,7 @@ impl Connection {
#[cfg(not(target_family = "wasm"))]
pub fn load_extension<P: AsRef<std::ffi::OsStr>>(&self, path: P) -> Result<()> {
Database::load_extension(self.db.as_ref(), path)
Database::load_extension(&self.db, path)
}
/// Close a connection and checkpoint.
@ -501,14 +507,75 @@ impl Statement {
}
}
pub type Row = types::OwnedRecord;
pub type Row = types::Record;
pub type StepResult = vdbe::StepResult;
#[derive(Clone, Debug)]
pub struct VirtualTable {
name: String,
args: Option<Vec<ast::Expr>>,
pub implementation: Rc<VTabModuleImpl>,
columns: Vec<Column>,
}
impl VirtualTable {
pub fn open(&self) -> VTabOpaqueCursor {
let cursor = unsafe { (self.implementation.open)() };
VTabOpaqueCursor::new(cursor)
}
pub fn filter(
&self,
cursor: &VTabOpaqueCursor,
arg_count: usize,
args: Vec<OwnedValue>,
) -> Result<()> {
let mut filter_args = Vec::with_capacity(arg_count);
for i in 0..arg_count {
let ownedvalue_arg = args.get(i).unwrap();
let extvalue_arg: ExtValue = match ownedvalue_arg {
OwnedValue::Null => Ok(ExtValue::null()),
OwnedValue::Integer(i) => Ok(ExtValue::from_integer(*i)),
OwnedValue::Float(f) => Ok(ExtValue::from_float(*f)),
OwnedValue::Text(t) => Ok(ExtValue::from_text(t.as_str().to_string())),
OwnedValue::Blob(b) => Ok(ExtValue::from_blob((**b).clone())),
other => Err(LimboError::ExtensionError(format!(
"Unsupported value type: {:?}",
other
))),
}?;
filter_args.push(extvalue_arg);
}
let rc = unsafe {
(self.implementation.filter)(cursor.as_ptr(), arg_count as i32, filter_args.as_ptr())
};
match rc {
ResultCode::OK => Ok(()),
_ => Err(LimboError::ExtensionError(rc.to_string())),
}
}
pub fn column(&self, cursor: &VTabOpaqueCursor, column: usize) -> Result<OwnedValue> {
let val = unsafe { (self.implementation.column)(cursor.as_ptr(), column as u32) };
OwnedValue::from_ffi(&val)
}
pub fn next(&self, cursor: &VTabOpaqueCursor) -> Result<bool> {
let rc = unsafe { (self.implementation.next)(cursor.as_ptr()) };
match rc {
ResultCode::OK => Ok(true),
ResultCode::EOF => Ok(false),
_ => Err(LimboError::ExtensionError("Next failed".to_string())),
}
}
}
pub(crate) struct SymbolTable {
pub functions: HashMap<String, Rc<function::ExternalFunc>>,
#[cfg(not(target_family = "wasm"))]
extensions: Vec<(Library, *const ExtensionApi)>,
pub vtabs: HashMap<String, VirtualTable>,
}
impl std::fmt::Debug for SymbolTable {
@ -550,6 +617,7 @@ impl SymbolTable {
pub fn new() -> Self {
Self {
functions: HashMap::new(),
vtabs: HashMap::new(),
#[cfg(not(target_family = "wasm"))]
extensions: Vec::new(),
}

8
core/pseudo.rs
View file

@ -1,7 +1,7 @@
use crate::types::OwnedRecord;
use crate::types::Record;
pub struct PseudoCursor {
current: Option<OwnedRecord>,
current: Option<Record>,
}
impl PseudoCursor {
@ -9,11 +9,11 @@ impl PseudoCursor {
Self { current: None }
}
pub fn record(&self) -> Option<&OwnedRecord> {
pub fn record(&self) -> Option<&Record> {
self.current.as_ref()
}
pub fn insert(&mut self, record: OwnedRecord) {
pub fn insert(&mut self, record: Record) {
self.current = Some(record);
}
}

29
core/schema.rs
View file

@ -1,3 +1,4 @@
use crate::VirtualTable;
use crate::{util::normalize_ident, Result};
use core::fmt;
use fallible_iterator::FallibleIterator;
@ -47,6 +48,7 @@ impl Schema {
pub enum Table {
BTree(Rc<BTreeTable>),
Pseudo(Rc<PseudoTable>),
Virtual(Rc<VirtualTable>),
}
impl Table {
@ -54,6 +56,7 @@ impl Table {
match self {
Table::BTree(table) => table.root_page,
Table::Pseudo(_) => unimplemented!(),
Table::Virtual(_) => unimplemented!(),
}
}
@ -61,19 +64,15 @@ impl Table {
match self {
Self::BTree(table) => &table.name,
Self::Pseudo(_) => "",
Self::Virtual(table) => &table.name,
}
}
pub fn get_column_at(&self, index: usize) -> &Column {
pub fn get_column_at(&self, index: usize) -> Option<&Column> {
match self {
Self::BTree(table) => table
.columns
.get(index)
.expect("column index out of bounds"),
Self::Pseudo(table) => table
.columns
.get(index)
.expect("column index out of bounds"),
Self::BTree(table) => table.columns.get(index),
Self::Pseudo(table) => table.columns.get(index),
Self::Virtual(table) => table.columns.get(index),
}
}
@ -81,6 +80,7 @@ impl Table {
match self {
Self::BTree(table) => &table.columns,
Self::Pseudo(table) => &table.columns,
Self::Virtual(table) => &table.columns,
}
}
@ -88,6 +88,14 @@ impl Table {
match self {
Self::BTree(table) => Some(table.clone()),
Self::Pseudo(_) => None,
Self::Virtual(_) => None,
}
}
pub fn virtual_table(&self) -> Option<Rc<VirtualTable>> {
match self {
Self::Virtual(table) => Some(table.clone()),
_ => None,
}
}
}
@ -97,6 +105,7 @@ impl PartialEq for Table {
match (self, other) {
(Self::BTree(a), Self::BTree(b)) => Rc::ptr_eq(a, b),
(Self::Pseudo(a), Self::Pseudo(b)) => Rc::ptr_eq(a, b),
(Self::Virtual(a), Self::Virtual(b)) => Rc::ptr_eq(a, b),
_ => false,
}
}
@ -155,7 +164,7 @@ impl BTreeTable {
sql.push_str(",\n");
}
sql.push_str(" ");
sql.push_str(&column.name.as_ref().expect("column name is None"));
sql.push_str(column.name.as_ref().expect("column name is None"));
sql.push(' ');
sql.push_str(&column.ty.to_string());
}

26
core/storage/btree.rs
View file

@ -5,7 +5,8 @@ use crate::storage::sqlite3_ondisk::{
read_btree_cell, read_varint, write_varint, BTreeCell, DatabaseHeader, PageContent, PageType,
TableInteriorCell, TableLeafCell,
};
use crate::types::{CursorResult, OwnedRecord, OwnedValue, SeekKey, SeekOp};
use crate::types::{CursorResult, OwnedValue, Record, SeekKey, SeekOp};
use crate::{LimboError, Result};
use std::cell::{Ref, RefCell};
@ -138,7 +139,7 @@ pub struct BTreeCursor {
root_page: usize,
/// Rowid and record are stored before being consumed.
rowid: RefCell<Option<u64>>,
record: RefCell<Option<OwnedRecord>>,
record: RefCell<Option<Record>>,
null_flag: bool,
/// Index internal pages are consumed on the way up, so we store going upwards flag in case
/// we just moved to a parent page and the parent page is an internal index page which requires
@ -199,7 +200,7 @@ impl BTreeCursor {
/// Move the cursor to the previous record and return it.
/// Used in backwards iteration.
fn get_prev_record(&mut self) -> Result<CursorResult<(Option<u64>, Option<OwnedRecord>)>> {
fn get_prev_record(&mut self) -> Result<CursorResult<(Option<u64>, Option<Record>)>> {
loop {
let page = self.stack.top();
let cell_idx = self.stack.current_cell_index();
@ -267,8 +268,7 @@ impl BTreeCursor {
_rowid, _payload, ..
}) => {
self.stack.retreat();
let record: OwnedRecord =
crate::storage::sqlite3_ondisk::read_record(&_payload)?;
let record: Record = crate::storage::sqlite3_ondisk::read_record(&_payload)?;
return Ok(CursorResult::Ok((Some(_rowid), Some(record))));
}
BTreeCell::IndexInteriorCell(_) => todo!(),
@ -282,7 +282,7 @@ impl BTreeCursor {
fn get_next_record(
&mut self,
predicate: Option<(SeekKey<'_>, SeekOp)>,
) -> Result<CursorResult<(Option<u64>, Option<OwnedRecord>)>> {
) -> Result<CursorResult<(Option<u64>, Option<Record>)>> {
loop {
let mem_page_rc = self.stack.top();
let cell_idx = self.stack.current_cell_index() as usize;
@ -445,7 +445,7 @@ impl BTreeCursor {
&mut self,
key: SeekKey<'_>,
op: SeekOp,
) -> Result<CursorResult<(Option<u64>, Option<OwnedRecord>)>> {
) -> Result<CursorResult<(Option<u64>, Option<Record>)>> {
return_if_io!(self.move_to(key.clone(), op.clone()));
{
@ -699,11 +699,7 @@ impl BTreeCursor {
/// Insert a record into the btree.
/// If the insert operation overflows the page, it will be split and the btree will be balanced.
fn insert_into_page(
&mut self,
key: &OwnedValue,
record: &OwnedRecord,
) -> Result<CursorResult<()>> {
fn insert_into_page(&mut self, key: &OwnedValue, record: &Record) -> Result<CursorResult<()>> {
if let CursorState::None = &self.state {
self.state = CursorState::Write(WriteInfo::new());
}
@ -1713,7 +1709,7 @@ impl BTreeCursor {
page_type: PageType,
int_key: Option<u64>,
cell_payload: &mut Vec<u8>,
record: &OwnedRecord,
record: &Record,
) {
assert!(matches!(
page_type,
@ -1947,14 +1943,14 @@ impl BTreeCursor {
Ok(CursorResult::Ok(rowid.is_some()))
}
pub fn record(&self) -> Result<Ref<Option<OwnedRecord>>> {
pub fn record(&self) -> Result<Ref<Option<Record>>> {
Ok(self.record.borrow())
}
pub fn insert(
&mut self,
key: &OwnedValue,
_record: &OwnedRecord,
_record: &Record,
moved_before: bool, /* Indicate whether it's necessary to traverse to find the leaf page */
) -> Result<CursorResult<()>> {
let int_key = match key {

15
core/storage/sqlite3_ondisk.rs
View file

@ -46,7 +46,7 @@ use crate::io::{Buffer, Completion, ReadCompletion, SyncCompletion, WriteComplet
use crate::storage::buffer_pool::BufferPool;
use crate::storage::database::DatabaseStorage;
use crate::storage::pager::Pager;
use crate::types::{OwnedRecord, OwnedValue};
use crate::types::{OwnedValue, Record, Text, TextSubtype};
use crate::{File, Result};
use log::trace;
use parking_lot::RwLock;
@ -948,7 +948,7 @@ impl TryFrom<u64> for SerialType {
}
}
pub fn read_record(payload: &[u8]) -> Result<OwnedRecord> {
pub fn read_record(payload: &[u8]) -> Result<Record> {
let mut pos = 0;
let (header_size, nr) = read_varint(payload)?;
assert!((header_size as usize) >= nr);
@ -969,7 +969,7 @@ pub fn read_record(payload: &[u8]) -> Result<OwnedRecord> {
pos += n;
values.push(value);
}
Ok(OwnedRecord::new(values))
Ok(Record::new(values))
}
pub fn read_value(buf: &[u8], serial_type: &SerialType) -> Result<(OwnedValue, usize)> {
@ -1059,8 +1059,13 @@ pub fn read_value(buf: &[u8], serial_type: &SerialType) -> Result<(OwnedValue, u
);
}
let bytes = buf[0..n].to_vec();
let value = unsafe { String::from_utf8_unchecked(bytes) };
Ok((OwnedValue::build_text(value.into()), n))
Ok((
OwnedValue::Text(Text {
value: Rc::new(bytes),
subtype: TextSubtype::Text,
}),
n,
))
}
}
}

42
core/translate/expr.rs
View file

@ -3,7 +3,7 @@ use sqlite3_parser::ast::{self, UnaryOperator};
#[cfg(feature = "json")]
use crate::function::JsonFunc;
use crate::function::{Func, FuncCtx, MathFuncArity, ScalarFunc, VectorFunc};
use crate::schema::Type;
use crate::schema::{Table, Type};
use crate::util::normalize_ident;
use crate::vdbe::{
builder::ProgramBuilder,
@ -1823,24 +1823,38 @@ pub fn translate_expr(
match table_reference.op {
// If we are reading a column from a table, we find the cursor that corresponds to
// the table and read the column from the cursor.
Operation::Scan { .. } | Operation::Search(_) => {
let cursor_id = program.resolve_cursor_id(&table_reference.identifier);
if *is_rowid_alias {
program.emit_insn(Insn::RowId {
cursor_id,
dest: target_register,
});
} else {
program.emit_insn(Insn::Column {
Operation::Scan { .. } | Operation::Search(_) => match &table_reference.table {
Table::BTree(_) => {
let cursor_id = program.resolve_cursor_id(&table_reference.identifier);
if *is_rowid_alias {
program.emit_insn(Insn::RowId {
cursor_id,
dest: target_register,
});
} else {
program.emit_insn(Insn::Column {
cursor_id,
column: *column,
dest: target_register,
});
}
let Some(column) = table_reference.table.get_column_at(*column) else {
crate::bail_parse_error!("column index out of bounds");
};
maybe_apply_affinity(column.ty, target_register, program);
Ok(target_register)
}
Table::Virtual(_) => {
let cursor_id = program.resolve_cursor_id(&table_reference.identifier);
program.emit_insn(Insn::VColumn {
cursor_id,
column: *column,
dest: target_register,
});
Ok(target_register)
}
let column = table_reference.table.get_column_at(*column);
maybe_apply_affinity(column.ty, target_register, program);
Ok(target_register)
}
_ => unreachable!(),
},
// If we are reading a column from a subquery, we instead copy the column from the
// subquery's result registers.
Operation::Subquery {

4
core/translate/group_by.rs
View file

@ -5,7 +5,7 @@ use sqlite3_parser::ast;
use crate::{
function::AggFunc,
schema::{Column, PseudoTable},
types::{OwnedRecord, OwnedValue},
types::{OwnedValue, Record},
vdbe::{
builder::{CursorType, ProgramBuilder},
insn::Insn,
@ -72,7 +72,7 @@ pub fn init_group_by(
program.emit_insn(Insn::SorterOpen {
cursor_id: sort_cursor,
columns: aggregates.len() + group_by.exprs.len(),
order: OwnedRecord::new(order),
order: Record::new(order),
});
program.add_comment(program.offset(), "clear group by abort flag");

137
core/translate/main_loop.rs
View file

@ -1,6 +1,7 @@
use sqlite3_parser::ast;
use crate::{
schema::Table,
translate::result_row::emit_select_result,
vdbe::{
builder::{CursorType, ProgramBuilder},
@ -80,25 +81,35 @@ pub fn init_loop(
Operation::Scan { .. } => {
let cursor_id = program.alloc_cursor_id(
Some(table.identifier.clone()),
CursorType::BTreeTable(table.btree().unwrap().clone()),
match &table.table {
Table::BTree(_) => CursorType::BTreeTable(table.btree().unwrap().clone()),
Table::Virtual(_) => {
CursorType::VirtualTable(table.virtual_table().unwrap().clone())
}
other => panic!("Invalid table reference type in Scan: {:?}", other),
},
);
let root_page = table.table.get_root_page();
match mode {
OperationMode::SELECT => {
match (mode, &table.table) {
(OperationMode::SELECT, Table::BTree(_)) => {
let root_page = table.btree().unwrap().root_page;
program.emit_insn(Insn::OpenReadAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenReadAwait {});
}
OperationMode::DELETE => {
(OperationMode::DELETE, Table::BTree(_)) => {
let root_page = table.btree().unwrap().root_page;
program.emit_insn(Insn::OpenWriteAsync {
cursor_id,
root_page,
});
program.emit_insn(Insn::OpenWriteAwait {});
}
(OperationMode::SELECT, Table::Virtual(_)) => {
program.emit_insn(Insn::VOpenAsync { cursor_id });
program.emit_insn(Insn::VOpenAwait {});
}
_ => {
unimplemented!()
}
@ -246,30 +257,55 @@ pub fn open_loop(
}
Operation::Scan { iter_dir } => {
let cursor_id = program.resolve_cursor_id(&table.identifier);
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
program.emit_insn(Insn::LastAsync { cursor_id });
} else {
program.emit_insn(Insn::RewindAsync { cursor_id });
}
program.emit_insn(
if !matches!(&table.table, Table::Virtual(_)) {
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
Insn::LastAwait {
cursor_id,
pc_if_empty: loop_end,
}
program.emit_insn(Insn::LastAsync { cursor_id });
} else {
Insn::RewindAwait {
cursor_id,
pc_if_empty: loop_end,
program.emit_insn(Insn::RewindAsync { cursor_id });
}
}
match &table.table {
Table::BTree(_) => program.emit_insn(
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
Insn::LastAwait {
cursor_id,
pc_if_empty: loop_end,
}
} else {
Insn::RewindAwait {
cursor_id,
pc_if_empty: loop_end,
}
},
),
Table::Virtual(ref table) => {
let args = if let Some(args) = table.args.as_ref() {
args
} else {
&vec![]
};
let start_reg = program.alloc_registers(args.len());
let mut cur_reg = start_reg;
for arg in args {
let reg = cur_reg;
cur_reg += 1;
translate_expr(program, Some(tables), &arg, reg, &t_ctx.resolver)?;
}
},
);
program.emit_insn(Insn::VFilter {
cursor_id,
arg_count: args.len(),
args_reg: start_reg,
});
}
other => panic!("Unsupported table reference type: {:?}", other),
}
program.resolve_label(loop_start, program.offset());
for cond in predicates
@ -690,27 +726,38 @@ pub fn close_loop(
Operation::Scan { iter_dir, .. } => {
program.resolve_label(loop_labels.next, program.offset());
let cursor_id = program.resolve_cursor_id(&table.identifier);
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
program.emit_insn(Insn::PrevAsync { cursor_id });
} else {
program.emit_insn(Insn::NextAsync { cursor_id });
}
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
program.emit_insn(Insn::PrevAwait {
cursor_id,
pc_if_next: loop_labels.loop_start,
});
} else {
program.emit_insn(Insn::NextAwait {
cursor_id,
pc_if_next: loop_labels.loop_start,
});
match &table.table {
Table::BTree(_) => {
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
program.emit_insn(Insn::PrevAsync { cursor_id });
} else {
program.emit_insn(Insn::NextAsync { cursor_id });
}
if iter_dir
.as_ref()
.is_some_and(|dir| *dir == IterationDirection::Backwards)
{
program.emit_insn(Insn::PrevAwait {
cursor_id,
pc_if_next: loop_labels.loop_start,
});
} else {
program.emit_insn(Insn::NextAwait {
cursor_id,
pc_if_next: loop_labels.loop_start,
});
}
}
Table::Virtual(_) => {
program.emit_insn(Insn::VNext {
cursor_id,
pc_if_next: loop_labels.loop_start,
});
}
other => unreachable!("Unsupported table reference type: {:?}", other),
}
}
Operation::Search(search) => {

30
core/translate/optimizer.rs
View file

@ -204,16 +204,16 @@ fn eliminate_constant_conditions(
}
fn push_scan_direction(table: &mut TableReference, direction: &Direction) {
match &mut table.op {
Operation::Scan { iter_dir, .. } => {
if iter_dir.is_none() {
match direction {
Direction::Ascending => *iter_dir = Some(IterationDirection::Forwards),
Direction::Descending => *iter_dir = Some(IterationDirection::Backwards),
}
if let Operation::Scan {
ref mut iter_dir, ..
} = table.op
{
if iter_dir.is_none() {
match direction {
Direction::Ascending => *iter_dir = Some(IterationDirection::Forwards),
Direction::Descending => *iter_dir = Some(IterationDirection::Backwards),
}
}
_ => {}
}
}
@ -307,14 +307,14 @@ impl Optimizable for ast::Expr {
else {
return Ok(None);
};
let column = table_reference.table.get_column_at(*column);
let Some(column) = table_reference.table.get_column_at(*column) else {
return Ok(None);
};
for index in available_indexes_for_table.iter() {
if column
.name
.as_ref()
.map_or(false, |name| *name == index.columns.first().unwrap().name)
{
return Ok(Some(index.clone()));
if let Some(name) = column.name.as_ref() {
if &index.columns.first().unwrap().name == name {
return Ok(Some(index.clone()));
}
}
}
Ok(None)

4
core/translate/order_by.rs
View file

@ -4,7 +4,7 @@ use sqlite3_parser::ast;
use crate::{
schema::{Column, PseudoTable},
types::{OwnedRecord, OwnedValue},
types::{OwnedValue, Record},
util::exprs_are_equivalent,
vdbe::{
builder::{CursorType, ProgramBuilder},
@ -47,7 +47,7 @@ pub fn init_order_by(
program.emit_insn(Insn::SorterOpen {
cursor_id: sort_cursor,
columns: order_by.len(),
order: OwnedRecord::new(order),
order: Record::new(order),
});
Ok(())
}

15
core/translate/plan.rs
View file

@ -9,6 +9,7 @@ use crate::{
function::AggFunc,
schema::{BTreeTable, Column, Index, Table},
vdbe::BranchOffset,
VirtualTable,
};
use crate::{
schema::{PseudoTable, Type},
@ -199,9 +200,6 @@ pub struct TableReference {
pub join_info: Option<JoinInfo>,
}
/**
A SourceOperator is a reference in the query plan that reads data from a table.
*/
#[derive(Clone, Debug)]
pub enum Operation {
// Scan operation
@ -229,7 +227,16 @@ pub enum Operation {
impl TableReference {
/// Returns the btree table for this table reference, if it is a BTreeTable.
pub fn btree(&self) -> Option<Rc<BTreeTable>> {
self.table.btree()
match &self.table {
Table::BTree(_) => self.table.btree(),
_ => None,
}
}
pub fn virtual_table(&self) -> Option<Rc<VirtualTable>> {
match &self.table {
Table::Virtual(_) => self.table.virtual_table(),
_ => None,
}
}
/// Creates a new TableReference for a subquery.

34
core/translate/planner.rs
View file

@ -11,7 +11,7 @@ use crate::{
schema::{Schema, Table},
util::{exprs_are_equivalent, normalize_ident},
vdbe::BranchOffset,
Result,
Result, VirtualTable,
};
use sqlite3_parser::ast::{self, Expr, FromClause, JoinType, Limit, UnaryOperator};
@ -317,8 +317,36 @@ fn parse_from_clause_table(
ast::As::Elided(id) => id.0.clone(),
})
.unwrap_or(format!("subquery_{}", cur_table_index));
let table_reference = TableReference::new_subquery(identifier, subplan, None);
Ok(table_reference)
Ok(TableReference::new_subquery(identifier, subplan, None))
}
ast::SelectTable::TableCall(qualified_name, maybe_args, maybe_alias) => {
let normalized_name = &normalize_ident(qualified_name.name.0.as_str());
let Some(vtab) = syms.vtabs.get(normalized_name) else {
crate::bail_parse_error!("Virtual table {} not found", normalized_name);
};
let alias = maybe_alias
.as_ref()
.map(|a| match a {
ast::As::As(id) => id.0.clone(),
ast::As::Elided(id) => id.0.clone(),
})
.unwrap_or(normalized_name.to_string());
Ok(TableReference {
op: Operation::Scan { iter_dir: None },
join_info: None,
table: Table::Virtual(
VirtualTable {
name: normalized_name.clone(),
args: maybe_args,
implementation: vtab.implementation.clone(),
columns: vtab.columns.clone(),
}
.into(),
)
.into(),
identifier: alias.clone(),
})
}
_ => todo!(),
}

6
core/translate/select.rs
View file

@ -28,9 +28,9 @@ pub fn translate_select(
let mut program = ProgramBuilder::new(ProgramBuilderOpts {
query_mode,
num_cursors: count_plan_required_cursors(&select),
approx_num_insns: estimate_num_instructions(&select),
approx_num_labels: estimate_num_labels(&select),
num_cursors: count_plan_required_cursors(select),
approx_num_insns: estimate_num_instructions(select),
approx_num_labels: estimate_num_labels(select),
});
emit_program(&mut program, select_plan, syms)?;
Ok(program)

69
core/types.rs
View file

@ -6,6 +6,7 @@ use crate::pseudo::PseudoCursor;
use crate::storage::btree::BTreeCursor;
use crate::storage::sqlite3_ondisk::write_varint;
use crate::vdbe::sorter::Sorter;
use crate::vdbe::VTabOpaqueCursor;
use crate::Result;
use std::fmt::Display;
use std::rc::Rc;
@ -49,7 +50,7 @@ pub enum TextSubtype {
#[derive(Debug, Clone, PartialEq)]
pub struct Text {
pub value: Rc<String>,
pub value: Rc<Vec<u8>>,
pub subtype: TextSubtype,
}
@ -60,17 +61,21 @@ impl Text {
pub fn new(value: Rc<String>) -> Self {
Self {
value,
value: Rc::new(value.as_bytes().to_vec()),
subtype: TextSubtype::Text,
}
}
pub fn json(value: Rc<String>) -> Self {
Self {
value,
value: Rc::new(value.as_bytes().to_vec()),
subtype: TextSubtype::Json,
}
}
pub fn as_str(&self) -> &str {
unsafe { std::str::from_utf8_unchecked(self.value.as_ref()) }
}
}
#[derive(Debug, Clone, PartialEq)]
@ -81,7 +86,7 @@ pub enum OwnedValue {
Text(Text),
Blob(Rc<Vec<u8>>),
Agg(Box<AggContext>), // TODO(pere): make this without Box. Currently this might cause cache miss but let's leave it for future analysis
Record(OwnedRecord),
Record(Record),
}
impl OwnedValue {
@ -103,7 +108,7 @@ impl OwnedValue {
pub fn to_text(&self) -> Option<&str> {
match self {
OwnedValue::Text(t) => Some(&t.value),
OwnedValue::Text(t) => Some(t.as_str()),
_ => None,
}
}
@ -129,7 +134,7 @@ impl OwnedValue {
OwnedValue::Null => Value::Null,
OwnedValue::Integer(i) => Value::Integer(*i),
OwnedValue::Float(f) => Value::Float(*f),
OwnedValue::Text(s) => Value::Text(&s.value),
OwnedValue::Text(s) => Value::Text(s.as_str()),
OwnedValue::Blob(b) => Value::Blob(b),
OwnedValue::Agg(a) => match a.as_ref() {
AggContext::Avg(acc, _count) => match acc {
@ -187,7 +192,7 @@ impl Display for OwnedValue {
Self::Null => write!(f, "NULL"),
Self::Integer(i) => write!(f, "{}", i),
Self::Float(fl) => write!(f, "{:?}", fl),
Self::Text(s) => write!(f, "{}", s.value),
Self::Text(s) => write!(f, "{}", s.as_str()),
Self::Blob(b) => write!(f, "{}", String::from_utf8_lossy(b)),
Self::Agg(a) => match a.as_ref() {
AggContext::Avg(acc, _count) => write!(f, "{}", acc),
@ -211,7 +216,7 @@ impl OwnedValue {
Self::Null => ExtValue::null(),
Self::Integer(i) => ExtValue::from_integer(*i),
Self::Float(fl) => ExtValue::from_float(*fl),
Self::Text(text) => ExtValue::from_text(text.value.to_string()),
Self::Text(text) => ExtValue::from_text(text.as_str().to_string()),
Self::Blob(blob) => ExtValue::from_blob(blob.to_vec()),
Self::Agg(_) => todo!(),
Self::Record(_) => todo!("Record values not yet supported"),
@ -377,21 +382,21 @@ impl std::ops::Add<OwnedValue> for OwnedValue {
Self::Float(float_left + float_right)
}
(Self::Text(string_left), Self::Text(string_right)) => Self::build_text(Rc::new(
string_left.value.to_string() + &string_right.value.to_string(),
string_left.as_str().to_string() + &string_right.as_str(),
)),
(Self::Text(string_left), Self::Integer(int_right)) => Self::build_text(Rc::new(
string_left.value.to_string() + &int_right.to_string(),
)),
(Self::Integer(int_left), Self::Text(string_right)) => Self::build_text(Rc::new(
int_left.to_string() + &string_right.value.to_string(),
string_left.as_str().to_string() + &int_right.to_string(),
)),
(Self::Integer(int_left), Self::Text(string_right)) => {
Self::build_text(Rc::new(int_left.to_string() + &string_right.as_str()))
}
(Self::Text(string_left), Self::Float(float_right)) => {
let string_right = Self::Float(float_right).to_string();
Self::build_text(Rc::new(string_left.value.to_string() + &string_right))
Self::build_text(Rc::new(string_left.as_str().to_string() + &string_right))
}
(Self::Float(float_left), Self::Text(string_right)) => {
let string_left = Self::Float(float_left).to_string();
Self::build_text(Rc::new(string_left + &string_right.value.to_string()))
Self::build_text(Rc::new(string_left + &string_right.as_str()))
}
(lhs, Self::Null) => lhs,
(Self::Null, rhs) => rhs,
@ -500,7 +505,7 @@ impl<'a> FromValue<'a> for i64 {
impl<'a> FromValue<'a> for String {
fn from_value(value: &'a OwnedValue) -> Result<Self> {
match value {
OwnedValue::Text(s) => Ok(s.value.to_string()),
OwnedValue::Text(s) => Ok(s.as_str().to_string()),
_ => Err(LimboError::ConversionError("Expected text value".into())),
}
}
@ -509,18 +514,18 @@ impl<'a> FromValue<'a> for String {
impl<'a> FromValue<'a> for &'a str {
fn from_value(value: &'a OwnedValue) -> Result<Self> {
match value {
OwnedValue::Text(s) => Ok(s.value.as_str()),
OwnedValue::Text(s) => Ok(s.as_str()),
_ => Err(LimboError::ConversionError("Expected text value".into())),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct OwnedRecord {
pub struct Record {
pub values: Vec<OwnedValue>,
}
impl OwnedRecord {
impl Record {
pub fn get<'a, T: FromValue<'a> + 'a>(&'a self, idx: usize) -> Result<T> {
let value = &self.values[idx];
T::from_value(value)
@ -600,7 +605,7 @@ impl From<SerialType> for u64 {
}
}
impl OwnedRecord {
impl Record {
pub fn new(values: Vec<OwnedValue>) -> Self {
Self { values }
}
@ -635,7 +640,7 @@ impl OwnedRecord {
}
}
OwnedValue::Float(f) => buf.extend_from_slice(&f.to_be_bytes()),
OwnedValue::Text(t) => buf.extend_from_slice(t.value.as_bytes()),
OwnedValue::Text(t) => buf.extend_from_slice(&t.value),
OwnedValue::Blob(b) => buf.extend_from_slice(b),
// non serializable
OwnedValue::Agg(_) => unreachable!(),
@ -666,6 +671,7 @@ pub enum Cursor {
Index(BTreeCursor),
Pseudo(PseudoCursor),
Sorter(Sorter),
Virtual(VTabOpaqueCursor),
}
impl Cursor {
@ -712,6 +718,13 @@ impl Cursor {
_ => panic!("Cursor is not a sorter cursor"),
}
}
pub fn as_virtual_mut(&mut self) -> &mut VTabOpaqueCursor {
match self {
Self::Virtual(cursor) => cursor,
_ => panic!("Cursor is not a virtual cursor"),
}
}
}
pub enum CursorResult<T> {
@ -729,7 +742,7 @@ pub enum SeekOp {
#[derive(Clone, PartialEq, Debug)]
pub enum SeekKey<'a> {
TableRowId(u64),
IndexKey(&'a OwnedRecord),
IndexKey(&'a Record),
}
#[cfg(test)]
@ -739,7 +752,7 @@ mod tests {
#[test]
fn test_serialize_null() {
let record = OwnedRecord::new(vec![OwnedValue::Null]);
let record = Record::new(vec![OwnedValue::Null]);
let mut buf = Vec::new();
record.serialize(&mut buf);
@ -755,7 +768,7 @@ mod tests {
#[test]
fn test_serialize_integers() {
let record = OwnedRecord::new(vec![
let record = Record::new(vec![
OwnedValue::Integer(42), // Should use SERIAL_TYPE_I8
OwnedValue::Integer(1000), // Should use SERIAL_TYPE_I16
OwnedValue::Integer(1_000_000), // Should use SERIAL_TYPE_I24
@ -831,7 +844,7 @@ mod tests {
#[test]
fn test_serialize_float() {
#[warn(clippy::approx_constant)]
let record = OwnedRecord::new(vec![OwnedValue::Float(3.15555)]);
let record = Record::new(vec![OwnedValue::Float(3.15555)]);
let mut buf = Vec::new();
record.serialize(&mut buf);
@ -852,7 +865,7 @@ mod tests {
#[test]
fn test_serialize_text() {
let text = Rc::new("hello".to_string());
let record = OwnedRecord::new(vec![OwnedValue::Text(Text::new(text.clone()))]);
let record = Record::new(vec![OwnedValue::Text(Text::new(text.clone()))]);
let mut buf = Vec::new();
record.serialize(&mut buf);
@ -871,7 +884,7 @@ mod tests {
#[test]
fn test_serialize_blob() {
let blob = Rc::new(vec![1, 2, 3, 4, 5]);
let record = OwnedRecord::new(vec![OwnedValue::Blob(blob.clone())]);
let record = Record::new(vec![OwnedValue::Blob(blob.clone())]);
let mut buf = Vec::new();
record.serialize(&mut buf);
@ -890,7 +903,7 @@ mod tests {
#[test]
fn test_serialize_mixed_types() {
let text = Rc::new("test".to_string());
let record = OwnedRecord::new(vec![
let record = Record::new(vec![
OwnedValue::Null,
OwnedValue::Integer(42),
OwnedValue::Float(3.15),

76
core/util.rs
View file

@ -1,9 +1,8 @@
use sqlite3_parser::ast::{self, CreateTableBody, Expr, FunctionTail, Literal};
use std::{rc::Rc, sync::Arc};
use sqlite3_parser::ast::{Expr, FunctionTail, Literal};
use crate::{
schema::{self, Schema},
schema::{self, Column, Schema, Type},
Result, Statement, StepResult, IO,
};
@ -308,6 +307,77 @@ pub fn exprs_are_equivalent(expr1: &Expr, expr2: &Expr) -> bool {
}
}
pub fn columns_from_create_table_body(body: ast::CreateTableBody) -> Result<Vec<Column>, ()> {
let CreateTableBody::ColumnsAndConstraints { columns, .. } = body else {
return Err(());
};
Ok(columns
.into_iter()
.filter_map(|(name, column_def)| {
// if column_def.col_type includes HIDDEN, omit it for now
if let Some(data_type) = column_def.col_type.as_ref() {
if data_type.name.as_str().contains("HIDDEN") {
return None;
}
}
let column = Column {
name: Some(name.0),
ty: match column_def.col_type {
Some(ref data_type) => {
// https://www.sqlite.org/datatype3.html
let type_name = data_type.name.as_str().to_uppercase();
if type_name.contains("INT") {
Type::Integer
} else if type_name.contains("CHAR")
|| type_name.contains("CLOB")
|| type_name.contains("TEXT")
{
Type::Text
} else if type_name.contains("BLOB") || type_name.is_empty() {
Type::Blob
} else if type_name.contains("REAL")
|| type_name.contains("FLOA")
|| type_name.contains("DOUB")
{
Type::Real
} else {
Type::Numeric
}
}
None => Type::Null,
},
default: column_def
.constraints
.iter()
.find_map(|c| match &c.constraint {
sqlite3_parser::ast::ColumnConstraint::Default(val) => Some(val.clone()),
_ => None,
}),
notnull: column_def.constraints.iter().any(|c| {
matches!(
c.constraint,
sqlite3_parser::ast::ColumnConstraint::NotNull { .. }
)
}),
ty_str: column_def
.col_type
.clone()
.map(|t| t.name.to_string())
.unwrap_or_default(),
primary_key: column_def.constraints.iter().any(|c| {
matches!(
c.constraint,
sqlite3_parser::ast::ColumnConstraint::PrimaryKey { .. }
)
}),
is_rowid_alias: false,
};
Some(column)
})
.collect::<Vec<_>>())
}
#[cfg(test)]
pub mod tests {
use super::*;

6
core/vdbe/builder.rs
View file

@ -9,7 +9,7 @@ use crate::{
schema::{BTreeTable, Index, PseudoTable},
storage::sqlite3_ondisk::DatabaseHeader,
translate::plan::{ResultSetColumn, TableReference},
Connection,
Connection, VirtualTable,
};
use super::{BranchOffset, CursorID, Insn, InsnReference, Program};
@ -40,6 +40,7 @@ pub enum CursorType {
BTreeIndex(Rc<Index>),
Pseudo(Rc<PseudoTable>),
Sorter,
VirtualTable(Rc<VirtualTable>),
}
impl CursorType {
@ -406,6 +407,9 @@ impl ProgramBuilder {
Insn::IsNull { reg: _, target_pc } => {
resolve(target_pc, "IsNull");
}
Insn::VNext { pc_if_next, .. } => {
resolve(pc_if_next, "VNext");
}
_ => continue,
}
}

57
core/vdbe/explain.rs
View file

@ -363,6 +363,62 @@ pub fn insn_to_str(
0,
"".to_string(),
),
Insn::VOpenAsync { cursor_id } => (
"VOpenAsync",
*cursor_id as i32,
0,
0,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::VOpenAwait => (
"VOpenAwait",
0,
0,
0,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::VFilter {
cursor_id,
arg_count,
args_reg,
} => (
"VFilter",
*cursor_id as i32,
*arg_count as i32,
*args_reg as i32,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::VColumn {
cursor_id,
column,
dest,
} => (
"VColumn",
*cursor_id as i32,
*column as i32,
*dest as i32,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::VNext {
cursor_id,
pc_if_next,
} => (
"VNext",
*cursor_id as i32,
pc_if_next.to_debug_int(),
0,
OwnedValue::build_text(Rc::new("".to_string())),
0,
"".to_string(),
),
Insn::OpenPseudo {
cursor_id,
content_reg,
@ -423,6 +479,7 @@ pub fn insn_to_str(
name
}
CursorType::Sorter => None,
CursorType::VirtualTable(v) => v.columns.get(*column).unwrap().name.as_ref(),
};
(
"Column",

127
core/vdbe/insn.rs
View file

@ -3,7 +3,7 @@ use std::rc::Rc;
use super::{AggFunc, BranchOffset, CursorID, FuncCtx, PageIdx};
use crate::storage::wal::CheckpointMode;
use crate::types::{OwnedRecord, OwnedValue};
use crate::types::{OwnedValue, Record};
use limbo_macros::Description;
/// Flags provided to comparison instructions (e.g. Eq, Ne) which determine behavior related to NULL values.
@ -213,6 +213,35 @@ pub enum Insn {
// Await for the completion of open cursor.
OpenReadAwait,
/// Open a cursor for a virtual table.
VOpenAsync {
cursor_id: CursorID,
},
/// Await for the completion of open cursor for a virtual table.
VOpenAwait,
/// Initialize the position of the virtual table cursor.
VFilter {
cursor_id: CursorID,
arg_count: usize,
args_reg: usize,
},
/// Read a column from the current row of the virtual table cursor.
VColumn {
cursor_id: CursorID,
column: usize,
dest: usize,
},
/// Advance the virtual table cursor to the next row.
/// TODO: async
VNext {
cursor_id: CursorID,
pc_if_next: BranchOffset,
},
// Open a cursor for a pseudo-table that contains a single row.
OpenPseudo {
cursor_id: CursorID,
@ -418,7 +447,7 @@ pub enum Insn {
SorterOpen {
cursor_id: CursorID, // P1
columns: usize, // P2
order: OwnedRecord, // P4. 0 if ASC and 1 if DESC
order: Record, // P4. 0 if ASC and 1 if DESC
},
// Insert a row into the sorter.
@ -643,11 +672,11 @@ pub fn exec_add(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
| (OwnedValue::Integer(i), OwnedValue::Float(f)) => OwnedValue::Float(*f + *i as f64),
(OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null,
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_add(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) | (other, OwnedValue::Text(text)) => {
exec_add(&cast_text_to_numerical(&text.value), other)
exec_add(&cast_text_to_numerical(text.as_str()), other)
}
_ => todo!(),
}
@ -674,14 +703,14 @@ pub fn exec_subtract(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
(OwnedValue::Integer(lhs), OwnedValue::Float(rhs)) => OwnedValue::Float(*lhs as f64 - rhs),
(OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null,
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_subtract(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) => {
exec_subtract(&cast_text_to_numerical(&text.value), other)
exec_subtract(&cast_text_to_numerical(text.as_str()), other)
}
(other, OwnedValue::Text(text)) => {
exec_subtract(other, &cast_text_to_numerical(&text.value))
exec_subtract(other, &cast_text_to_numerical(text.as_str()))
}
_ => todo!(),
}
@ -707,11 +736,11 @@ pub fn exec_multiply(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
| (OwnedValue::Float(f), OwnedValue::Integer(i)) => OwnedValue::Float(*i as f64 * { *f }),
(OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null,
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_multiply(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) | (other, OwnedValue::Text(text)) => {
exec_multiply(&cast_text_to_numerical(&text.value), other)
exec_multiply(&cast_text_to_numerical(text.as_str()), other)
}
_ => todo!(),
@ -740,11 +769,15 @@ pub fn exec_divide(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
(OwnedValue::Integer(lhs), OwnedValue::Float(rhs)) => OwnedValue::Float(*lhs as f64 / rhs),
(OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null,
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_divide(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) => exec_divide(&cast_text_to_numerical(&text.value), other),
(other, OwnedValue::Text(text)) => exec_divide(other, &cast_text_to_numerical(&text.value)),
(OwnedValue::Text(text), other) => {
exec_divide(&cast_text_to_numerical(text.as_str()), other)
}
(other, OwnedValue::Text(text)) => {
exec_divide(other, &cast_text_to_numerical(text.as_str()))
}
_ => todo!(),
}
}
@ -769,11 +802,11 @@ pub fn exec_bit_and(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
(OwnedValue::Float(lh), OwnedValue::Integer(rh)) => OwnedValue::Integer(*lh as i64 & rh),
(OwnedValue::Integer(lh), OwnedValue::Float(rh)) => OwnedValue::Integer(lh & *rh as i64),
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_bit_and(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) | (other, OwnedValue::Text(text)) => {
exec_bit_and(&cast_text_to_numerical(&text.value), other)
exec_bit_and(&cast_text_to_numerical(text.as_str()), other)
}
_ => todo!(),
}
@ -795,11 +828,11 @@ pub fn exec_bit_or(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
OwnedValue::Integer(*lh as i64 | *rh as i64)
}
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_bit_or(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) | (other, OwnedValue::Text(text)) => {
exec_bit_or(&cast_text_to_numerical(&text.value), other)
exec_bit_or(&cast_text_to_numerical(text.as_str()), other)
}
_ => todo!(),
}
@ -839,7 +872,7 @@ pub fn exec_bit_not(mut reg: &OwnedValue) -> OwnedValue {
OwnedValue::Null => OwnedValue::Null,
OwnedValue::Integer(i) => OwnedValue::Integer(!i),
OwnedValue::Float(f) => OwnedValue::Integer(!(*f as i64)),
OwnedValue::Text(text) => exec_bit_not(&cast_text_to_numerical(&text.value)),
OwnedValue::Text(text) => exec_bit_not(&cast_text_to_numerical(text.as_str())),
_ => todo!(),
}
}
@ -866,14 +899,14 @@ pub fn exec_shift_left(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue
OwnedValue::Integer(compute_shl(*lh as i64, *rh as i64))
}
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_shift_left(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) => {
exec_shift_left(&cast_text_to_numerical(&text.value), other)
exec_shift_left(&cast_text_to_numerical(text.as_str()), other)
}
(other, OwnedValue::Text(text)) => {
exec_shift_left(other, &cast_text_to_numerical(&text.value))
exec_shift_left(other, &cast_text_to_numerical(text.as_str()))
}
_ => todo!(),
}
@ -905,14 +938,14 @@ pub fn exec_shift_right(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValu
OwnedValue::Integer(compute_shr(*lh as i64, *rh as i64))
}
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_shift_right(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) => {
exec_shift_right(&cast_text_to_numerical(&text.value), other)
exec_shift_right(&cast_text_to_numerical(text.as_str()), other)
}
(other, OwnedValue::Text(text)) => {
exec_shift_right(other, &cast_text_to_numerical(&text.value))
exec_shift_right(other, &cast_text_to_numerical(text.as_str()))
}
_ => todo!(),
}
@ -943,7 +976,7 @@ pub fn exec_boolean_not(mut reg: &OwnedValue) -> OwnedValue {
OwnedValue::Null => OwnedValue::Null,
OwnedValue::Integer(i) => OwnedValue::Integer((*i == 0) as i64),
OwnedValue::Float(f) => OwnedValue::Integer((*f == 0.0) as i64),
OwnedValue::Text(text) => exec_boolean_not(&cast_text_to_numerical(&text.value)),
OwnedValue::Text(text) => exec_boolean_not(&cast_text_to_numerical(text.as_str())),
_ => todo!(),
}
}
@ -951,20 +984,20 @@ pub fn exec_boolean_not(mut reg: &OwnedValue) -> OwnedValue {
pub fn exec_concat(lhs: &OwnedValue, rhs: &OwnedValue) -> OwnedValue {
match (lhs, rhs) {
(OwnedValue::Text(lhs_text), OwnedValue::Text(rhs_text)) => {
OwnedValue::build_text(Rc::new(lhs_text.value.as_ref().clone() + &rhs_text.value))
OwnedValue::build_text(Rc::new(lhs_text.as_str().to_string() + &rhs_text.as_str()))
}
(OwnedValue::Text(lhs_text), OwnedValue::Integer(rhs_int)) => OwnedValue::build_text(
Rc::new(lhs_text.value.as_ref().clone() + &rhs_int.to_string()),
Rc::new(lhs_text.as_str().to_string() + &rhs_int.to_string()),
),
(OwnedValue::Text(lhs_text), OwnedValue::Float(rhs_float)) => OwnedValue::build_text(
Rc::new(lhs_text.value.as_ref().clone() + &rhs_float.to_string()),
Rc::new(lhs_text.as_str().to_string() + &rhs_float.to_string()),
),
(OwnedValue::Text(lhs_text), OwnedValue::Agg(rhs_agg)) => OwnedValue::build_text(Rc::new(
lhs_text.value.as_ref().clone() + &rhs_agg.final_value().to_string(),
lhs_text.as_str().to_string() + &rhs_agg.final_value().to_string(),
)),
(OwnedValue::Integer(lhs_int), OwnedValue::Text(rhs_text)) => {
OwnedValue::build_text(Rc::new(lhs_int.to_string() + &rhs_text.value))
OwnedValue::build_text(Rc::new(lhs_int.to_string() + rhs_text.as_str()))
}
(OwnedValue::Integer(lhs_int), OwnedValue::Integer(rhs_int)) => {
OwnedValue::build_text(Rc::new(lhs_int.to_string() + &rhs_int.to_string()))
@ -977,7 +1010,7 @@ pub fn exec_concat(lhs: &OwnedValue, rhs: &OwnedValue) -> OwnedValue {
),
(OwnedValue::Float(lhs_float), OwnedValue::Text(rhs_text)) => {
OwnedValue::build_text(Rc::new(lhs_float.to_string() + &rhs_text.value))
OwnedValue::build_text(Rc::new(lhs_float.to_string() + rhs_text.as_str()))
}
(OwnedValue::Float(lhs_float), OwnedValue::Integer(rhs_int)) => {
OwnedValue::build_text(Rc::new(lhs_float.to_string() + &rhs_int.to_string()))
@ -989,9 +1022,9 @@ pub fn exec_concat(lhs: &OwnedValue, rhs: &OwnedValue) -> OwnedValue {
Rc::new(lhs_float.to_string() + &rhs_agg.final_value().to_string()),
),
(OwnedValue::Agg(lhs_agg), OwnedValue::Text(rhs_text)) => {
OwnedValue::build_text(Rc::new(lhs_agg.final_value().to_string() + &rhs_text.value))
}
(OwnedValue::Agg(lhs_agg), OwnedValue::Text(rhs_text)) => OwnedValue::build_text(Rc::new(
lhs_agg.final_value().to_string() + rhs_text.as_str(),
)),
(OwnedValue::Agg(lhs_agg), OwnedValue::Integer(rhs_int)) => OwnedValue::build_text(
Rc::new(lhs_agg.final_value().to_string() + &rhs_int.to_string()),
),
@ -1025,11 +1058,11 @@ pub fn exec_and(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
| (OwnedValue::Float(0.0), _) => OwnedValue::Integer(0),
(OwnedValue::Null, _) | (_, OwnedValue::Null) => OwnedValue::Null,
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_and(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) | (other, OwnedValue::Text(text)) => {
exec_and(&cast_text_to_numerical(&text.value), other)
exec_and(&cast_text_to_numerical(text.as_str()), other)
}
_ => OwnedValue::Integer(1),
}
@ -1054,11 +1087,11 @@ pub fn exec_or(mut lhs: &OwnedValue, mut rhs: &OwnedValue) -> OwnedValue {
| (OwnedValue::Float(0.0), OwnedValue::Float(0.0))
| (OwnedValue::Integer(0), OwnedValue::Integer(0)) => OwnedValue::Integer(0),
(OwnedValue::Text(lhs), OwnedValue::Text(rhs)) => exec_or(
&cast_text_to_numerical(&lhs.value),
&cast_text_to_numerical(&rhs.value),
&cast_text_to_numerical(lhs.as_str()),
&cast_text_to_numerical(rhs.as_str()),
),
(OwnedValue::Text(text), other) | (other, OwnedValue::Text(text)) => {
exec_or(&cast_text_to_numerical(&text.value), other)
exec_or(&cast_text_to_numerical(text.as_str()), other)
}
_ => OwnedValue::Integer(1),
}

2
core/vdbe/likeop.rs
View file

@ -7,7 +7,7 @@ use crate::{types::OwnedValue, LimboError};
pub fn construct_like_escape_arg(escape_value: &OwnedValue) -> Result<char, LimboError> {
match escape_value {
OwnedValue::Text(text) => {
let mut escape_chars = text.value.chars();
let mut escape_chars = text.as_str().chars();
match (escape_chars.next(), escape_chars.next()) {
(Some(escape), None) => Ok(escape),
_ => Err(LimboError::Constraint(

235
core/vdbe/mod.rs
View file

@ -18,17 +18,18 @@
//! https://www.sqlite.org/opcode.html
pub mod builder;
mod datetime;
pub mod explain;
pub mod insn;
pub mod likeop;
mod printf;
pub mod sorter;
mod strftime;
use crate::error::{LimboError, SQLITE_CONSTRAINT_PRIMARYKEY};
use crate::ext::ExtValue;
use crate::function::{AggFunc, ExtFunc, FuncCtx, MathFunc, MathFuncArity, ScalarFunc, VectorFunc};
use crate::functions::datetime::{
exec_date, exec_datetime_full, exec_julianday, exec_strftime, exec_time, exec_unixepoch,
};
use crate::functions::printf::exec_printf;
use crate::info;
use crate::pseudo::PseudoCursor;
use crate::result::LimboResult;
@ -37,7 +38,7 @@ use crate::storage::wal::CheckpointResult;
use crate::storage::{btree::BTreeCursor, pager::Pager};
use crate::translate::plan::{ResultSetColumn, TableReference};
use crate::types::{
AggContext, Cursor, CursorResult, ExternalAggState, OwnedRecord, OwnedValue, SeekKey, SeekOp,
AggContext, Cursor, CursorResult, ExternalAggState, OwnedValue, Record, SeekKey, SeekOp,
};
use crate::util::parse_schema_rows;
use crate::vdbe::builder::CursorType;
@ -51,16 +52,12 @@ use crate::{
json::json_remove, json::json_set, json::json_type,
};
use crate::{resolve_ext_path, Connection, Result, TransactionState, DATABASE_VERSION};
use datetime::{
exec_date, exec_datetime_full, exec_julianday, exec_strftime, exec_time, exec_unixepoch,
};
use insn::{
exec_add, exec_and, exec_bit_and, exec_bit_not, exec_bit_or, exec_boolean_not, exec_concat,
exec_divide, exec_multiply, exec_or, exec_remainder, exec_shift_left, exec_shift_right,
exec_subtract,
};
use likeop::{construct_like_escape_arg, exec_glob, exec_like_with_escape};
use printf::exec_printf;
use rand::distributions::{Distribution, Uniform};
use rand::{thread_rng, Rng};
use regex::{Regex, RegexBuilder};
@ -68,6 +65,7 @@ use sorter::Sorter;
use std::borrow::BorrowMut;
use std::cell::{Cell, RefCell, RefMut};
use std::collections::HashMap;
use std::ffi::c_void;
use std::num::NonZero;
use std::rc::{Rc, Weak};
@ -270,6 +268,19 @@ fn get_cursor_as_sorter_mut<'long, 'short>(
cursor
}
fn get_cursor_as_virtual_mut<'long, 'short>(
cursors: &'short mut RefMut<'long, Vec<Option<Cursor>>>,
cursor_id: CursorID,
) -> &'short mut VTabOpaqueCursor {
let cursor = cursors
.get_mut(cursor_id)
.expect("cursor id out of bounds")
.as_mut()
.expect("cursor not allocated")
.as_virtual_mut();
cursor
}
struct Bitfield<const N: usize>([u64; N]);
impl<const N: usize> Bitfield<N> {
@ -293,12 +304,24 @@ impl<const N: usize> Bitfield<N> {
}
}
pub struct VTabOpaqueCursor(*mut c_void);
impl VTabOpaqueCursor {
pub fn new(cursor: *mut c_void) -> Self {
Self(cursor)
}
pub fn as_ptr(&self) -> *mut c_void {
self.0
}
}
/// The program state describes the environment in which the program executes.
pub struct ProgramState {
pub pc: InsnReference,
cursors: RefCell<Vec<Option<Cursor>>>,
registers: Vec<OwnedValue>,
pub(crate) result_row: Option<OwnedRecord>,
pub(crate) result_row: Option<Record>,
last_compare: Option<std::cmp::Ordering>,
deferred_seek: Option<(CursorID, CursorID)>,
ended_coroutine: Bitfield<4>, // flag to indicate that a coroutine has ended (key is the yield register. currently we assume that the yield register is always between 0-255, YOLO)
@ -373,6 +396,7 @@ macro_rules! must_be_btree_cursor {
CursorType::BTreeIndex(_) => get_cursor_as_index_mut(&mut $cursors, $cursor_id),
CursorType::Pseudo(_) => panic!("{} on pseudo cursor", $insn_name),
CursorType::Sorter => panic!("{} on sorter cursor", $insn_name),
CursorType::VirtualTable(_) => panic!("{} on virtual table cursor", $insn_name),
};
cursor
}};
@ -829,12 +853,79 @@ impl Program {
CursorType::Sorter => {
panic!("OpenReadAsync on sorter cursor");
}
CursorType::VirtualTable(_) => {
panic!("OpenReadAsync on virtual table cursor, use Insn::VOpenAsync instead");
}
}
state.pc += 1;
}
Insn::OpenReadAwait => {
state.pc += 1;
}
Insn::VOpenAsync { cursor_id } => {
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
let CursorType::VirtualTable(virtual_table) = cursor_type else {
panic!("VOpenAsync on non-virtual table cursor");
};
let cursor = virtual_table.open();
state
.cursors
.borrow_mut()
.insert(*cursor_id, Some(Cursor::Virtual(cursor)));
state.pc += 1;
}
Insn::VOpenAwait => {
state.pc += 1;
}
Insn::VFilter {
cursor_id,
arg_count,
args_reg,
} => {
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
let CursorType::VirtualTable(virtual_table) = cursor_type else {
panic!("VFilter on non-virtual table cursor");
};
let mut cursors = state.cursors.borrow_mut();
let cursor = get_cursor_as_virtual_mut(&mut cursors, *cursor_id);
let mut args = Vec::new();
for i in 0..*arg_count {
args.push(state.registers[args_reg + i].clone());
}
virtual_table.filter(cursor, *arg_count, args)?;
state.pc += 1;
}
Insn::VColumn {
cursor_id,
column,
dest,
} => {
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
let CursorType::VirtualTable(virtual_table) = cursor_type else {
panic!("VColumn on non-virtual table cursor");
};
let mut cursors = state.cursors.borrow_mut();
let cursor = get_cursor_as_virtual_mut(&mut cursors, *cursor_id);
state.registers[*dest] = virtual_table.column(cursor, *column)?;
state.pc += 1;
}
Insn::VNext {
cursor_id,
pc_if_next,
} => {
let (_, cursor_type) = self.cursor_ref.get(*cursor_id).unwrap();
let CursorType::VirtualTable(virtual_table) = cursor_type else {
panic!("VNextAsync on non-virtual table cursor");
};
let mut cursors = state.cursors.borrow_mut();
let cursor = get_cursor_as_virtual_mut(&mut cursors, *cursor_id);
let has_more = virtual_table.next(cursor)?;
if has_more {
state.pc = pc_if_next.to_offset_int();
} else {
state.pc += 1;
}
}
Insn::OpenPseudo {
cursor_id,
content_reg: _,
@ -946,6 +1037,11 @@ impl Program {
state.registers[*dest] = OwnedValue::Null;
}
}
CursorType::VirtualTable(_) => {
panic!(
"Insn::Column on virtual table cursor, use Insn::VColumn instead"
);
}
}
state.pc += 1;
@ -1208,7 +1304,7 @@ impl Program {
let mut cursors = state.cursors.borrow_mut();
if *is_index {
let cursor = get_cursor_as_index_mut(&mut cursors, *cursor_id);
let record_from_regs: OwnedRecord =
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
let found = return_if_io!(
cursor.seek(SeekKey::IndexKey(&record_from_regs), SeekOp::GE)
@ -1254,7 +1350,7 @@ impl Program {
let mut cursors = state.cursors.borrow_mut();
if *is_index {
let cursor = get_cursor_as_index_mut(&mut cursors, *cursor_id);
let record_from_regs: OwnedRecord =
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
let found = return_if_io!(
cursor.seek(SeekKey::IndexKey(&record_from_regs), SeekOp::GT)
@ -1298,7 +1394,7 @@ impl Program {
assert!(target_pc.is_offset());
let mut cursors = state.cursors.borrow_mut();
let cursor = get_cursor_as_index_mut(&mut cursors, *cursor_id);
let record_from_regs: OwnedRecord =
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
// omit the rowid from the idx_record, which is the last value
@ -1322,7 +1418,7 @@ impl Program {
assert!(target_pc.is_offset());
let mut cursors = state.cursors.borrow_mut();
let cursor = get_cursor_as_index_mut(&mut cursors, *cursor_id);
let record_from_regs: OwnedRecord =
let record_from_regs: Record =
make_owned_record(&state.registers, start_reg, num_regs);
if let Some(ref idx_record) = *cursor.record()? {
// omit the rowid from the idx_record, which is the last value
@ -1840,12 +1936,12 @@ impl Program {
// To the way blobs are parsed here in SQLite.
let indent = match indent {
Some(value) => match value {
OwnedValue::Text(text) => text.value.as_str(),
OwnedValue::Text(text) => text.as_str(),
OwnedValue::Integer(val) => &val.to_string(),
OwnedValue::Float(val) => &val.to_string(),
OwnedValue::Blob(val) => &String::from_utf8_lossy(val),
OwnedValue::Agg(ctx) => match ctx.final_value() {
OwnedValue::Text(text) => text.value.as_str(),
OwnedValue::Text(text) => text.as_str(),
OwnedValue::Integer(val) => &val.to_string(),
OwnedValue::Float(val) => &val.to_string(),
OwnedValue::Blob(val) => &String::from_utf8_lossy(val),
@ -1883,7 +1979,8 @@ impl Program {
else {
unreachable!("Cast with non-text type");
};
let result = exec_cast(&reg_value_argument, &reg_value_type.value);
let result =
exec_cast(&reg_value_argument, &reg_value_type.as_str());
state.registers[*dest] = result;
}
ScalarFunc::Changes => {
@ -1921,8 +2018,8 @@ impl Program {
};
OwnedValue::Integer(exec_glob(
cache,
&pattern.value,
&text.value,
&pattern.as_str(),
&text.as_str(),
)
as i64)
}
@ -1964,8 +2061,8 @@ impl Program {
};
OwnedValue::Integer(exec_like_with_escape(
&pattern.value,
&text.value,
&pattern.as_str(),
&text.as_str(),
escape,
)
as i64)
@ -1978,8 +2075,8 @@ impl Program {
};
OwnedValue::Integer(exec_like(
cache,
&pattern.value,
&text.value,
&pattern.as_str(),
&text.as_str(),
)
as i64)
}
@ -2408,14 +2505,16 @@ impl Program {
"MustBeInt: the value in register cannot be cast to integer"
),
},
OwnedValue::Text(text) => match checked_cast_text_to_numeric(&text.value) {
Ok(OwnedValue::Integer(i)) => {
state.registers[*reg] = OwnedValue::Integer(i)
OwnedValue::Text(text) => {
match checked_cast_text_to_numeric(&text.as_str()) {
Ok(OwnedValue::Integer(i)) => {
state.registers[*reg] = OwnedValue::Integer(i)
}
_ => crate::bail_parse_error!(
"MustBeInt: the value in register cannot be cast to integer"
),
}
_ => crate::bail_parse_error!(
"MustBeInt: the value in register cannot be cast to integer"
),
},
}
_ => {
crate::bail_parse_error!(
"MustBeInt: the value in register cannot be cast to integer"
@ -2513,7 +2612,7 @@ impl Program {
}
Insn::CreateBtree { db, root, flags } => {
if *db > 0 {
// TODO: implement temp datbases
// TODO: implement temp databases
todo!("temp databases not implemented yet");
}
let mut cursor = Box::new(BTreeCursor::new(pager.clone(), 0));
@ -2652,12 +2751,12 @@ fn get_new_rowid<R: Rng>(cursor: &mut BTreeCursor, mut rng: R) -> Result<CursorR
Ok(CursorResult::Ok(rowid.try_into().unwrap()))
}
fn make_owned_record(registers: &[OwnedValue], start_reg: &usize, count: &usize) -> OwnedRecord {
fn make_owned_record(registers: &[OwnedValue], start_reg: &usize, count: &usize) -> Record {
let mut values = Vec::with_capacity(*count);
for r in registers.iter().skip(*start_reg).take(*count) {
values.push(r.clone())
}
OwnedRecord::new(values)
Record::new(values)
}
fn trace_insn(program: &Program, addr: InsnReference, insn: &Insn) {
@ -2717,7 +2816,7 @@ fn get_indent_count(indent_count: usize, curr_insn: &Insn, prev_insn: Option<&In
fn exec_lower(reg: &OwnedValue) -> Option<OwnedValue> {
match reg {
OwnedValue::Text(t) => Some(OwnedValue::build_text(Rc::new(t.value.to_lowercase()))),
OwnedValue::Text(t) => Some(OwnedValue::build_text(Rc::new(t.as_str().to_lowercase()))),
t => Some(t.to_owned()),
}
}
@ -2746,7 +2845,7 @@ fn exec_octet_length(reg: &OwnedValue) -> OwnedValue {
fn exec_upper(reg: &OwnedValue) -> Option<OwnedValue> {
match reg {
OwnedValue::Text(t) => Some(OwnedValue::build_text(Rc::new(t.value.to_uppercase()))),
OwnedValue::Text(t) => Some(OwnedValue::build_text(Rc::new(t.as_str().to_uppercase()))),
t => Some(t.to_owned()),
}
}
@ -2755,7 +2854,7 @@ fn exec_concat_strings(registers: &[OwnedValue]) -> OwnedValue {
let mut result = String::new();
for reg in registers {
match reg {
OwnedValue::Text(text) => result.push_str(&text.value),
OwnedValue::Text(text) => result.push_str(text.as_str()),
OwnedValue::Integer(i) => result.push_str(&i.to_string()),
OwnedValue::Float(f) => result.push_str(&f.to_string()),
OwnedValue::Agg(aggctx) => result.push_str(&aggctx.final_value().to_string()),
@ -2773,9 +2872,9 @@ fn exec_concat_ws(registers: &[OwnedValue]) -> OwnedValue {
}
let separator = match &registers[0] {
OwnedValue::Text(text) => text.value.clone(),
OwnedValue::Integer(i) => Rc::new(i.to_string()),
OwnedValue::Float(f) => Rc::new(f.to_string()),
OwnedValue::Text(text) => text.as_str().to_string(),
OwnedValue::Integer(i) => i.to_string(),
OwnedValue::Float(f) => f.to_string(),
_ => return OwnedValue::Null,
};
@ -2785,7 +2884,7 @@ fn exec_concat_ws(registers: &[OwnedValue]) -> OwnedValue {
result.push_str(&separator);
}
match reg {
OwnedValue::Text(text) => result.push_str(&text.value),
OwnedValue::Text(text) => result.push_str(text.as_str()),
OwnedValue::Integer(i) => result.push_str(&i.to_string()),
OwnedValue::Float(f) => result.push_str(&f.to_string()),
_ => continue,
@ -2800,9 +2899,9 @@ fn exec_sign(reg: &OwnedValue) -> Option<OwnedValue> {
OwnedValue::Integer(i) => *i as f64,
OwnedValue::Float(f) => *f,
OwnedValue::Text(s) => {
if let Ok(i) = s.value.parse::<i64>() {
if let Ok(i) = s.as_str().parse::<i64>() {
i as f64
} else if let Ok(f) = s.value.parse::<f64>() {
} else if let Ok(f) = s.as_str().parse::<f64>() {
f
} else {
return Some(OwnedValue::Null);
@ -2840,7 +2939,7 @@ pub fn exec_soundex(reg: &OwnedValue) -> OwnedValue {
OwnedValue::Null => return OwnedValue::build_text(Rc::new("?000".to_string())),
OwnedValue::Text(s) => {
// return ?000 if non ASCII alphabet character is found
if !s.value.chars().all(|c| c.is_ascii_alphabetic()) {
if !s.as_str().chars().all(|c| c.is_ascii_alphabetic()) {
return OwnedValue::build_text(Rc::new("?000".to_string()));
}
s.clone()
@ -2850,7 +2949,7 @@ pub fn exec_soundex(reg: &OwnedValue) -> OwnedValue {
// Remove numbers and spaces
let word: String = s
.value
.as_str()
.chars()
.filter(|c| !c.is_ascii_digit())
.collect::<String>()
@ -2958,7 +3057,7 @@ fn exec_randomblob(reg: &OwnedValue) -> OwnedValue {
let length = match reg {
OwnedValue::Integer(i) => *i,
OwnedValue::Float(f) => *f as i64,
OwnedValue::Text(t) => t.value.parse().unwrap_or(1),
OwnedValue::Text(t) => t.as_str().parse().unwrap_or(1),
_ => 1,
}
.max(1) as usize;
@ -2974,9 +3073,9 @@ fn exec_quote(value: &OwnedValue) -> OwnedValue {
OwnedValue::Integer(_) | OwnedValue::Float(_) => value.to_owned(),
OwnedValue::Blob(_) => todo!(),
OwnedValue::Text(s) => {
let mut quoted = String::with_capacity(s.value.len() + 2);
let mut quoted = String::with_capacity(s.as_str().len() + 2);
quoted.push('\'');
for c in s.value.chars() {
for c in s.as_str().chars() {
if c == '\0' {
break;
} else {
@ -3081,7 +3180,7 @@ fn exec_substring(
(str_value, start_value, length_value)
{
let start = *start as usize;
let str_len = str.value.len();
let str_len = str.as_str().len();
if start > str_len {
return OwnedValue::build_text(Rc::new("".to_string()));
@ -3093,19 +3192,19 @@ fn exec_substring(
} else {
str_len
};
let substring = &str.value[start_idx..end.min(str_len)];
let substring = &str.as_str()[start_idx..end.min(str_len)];
OwnedValue::build_text(Rc::new(substring.to_string()))
} else if let (OwnedValue::Text(str), OwnedValue::Integer(start)) = (str_value, start_value) {
let start = *start as usize;
let str_len = str.value.len();
let str_len = str.as_str().len();
if start > str_len {
return OwnedValue::build_text(Rc::new("".to_string()));
}
let start_idx = start - 1;
let substring = &str.value[start_idx..str_len];
let substring = &str.as_str()[start_idx..str_len];
OwnedValue::build_text(Rc::new(substring.to_string()))
} else {
@ -3128,7 +3227,7 @@ fn exec_instr(reg: &OwnedValue, pattern: &OwnedValue) -> OwnedValue {
let reg_str;
let reg = match reg {
OwnedValue::Text(s) => s.value.as_str(),
OwnedValue::Text(s) => s.as_str(),
_ => {
reg_str = reg.to_string();
reg_str.as_str()
@ -3137,7 +3236,7 @@ fn exec_instr(reg: &OwnedValue, pattern: &OwnedValue) -> OwnedValue {
let pattern_str;
let pattern = match pattern {
OwnedValue::Text(s) => s.value.as_str(),
OwnedValue::Text(s) => s.as_str(),
_ => {
pattern_str = pattern.to_string();
pattern_str.as_str()
@ -3221,7 +3320,7 @@ fn exec_unicode(reg: &OwnedValue) -> OwnedValue {
fn _to_float(reg: &OwnedValue) -> f64 {
match reg {
OwnedValue::Text(x) => x.value.parse().unwrap_or(0.0),
OwnedValue::Text(x) => x.as_str().parse().unwrap_or(0.0),
OwnedValue::Integer(x) => *x as f64,
OwnedValue::Float(x) => *x,
_ => 0.0,
@ -3230,7 +3329,7 @@ fn _to_float(reg: &OwnedValue) -> f64 {
fn exec_round(reg: &OwnedValue, precision: Option<OwnedValue>) -> OwnedValue {
let precision = match precision {
Some(OwnedValue::Text(x)) => x.value.parse().unwrap_or(0.0),
Some(OwnedValue::Text(x)) => x.as_str().parse().unwrap_or(0.0),
Some(OwnedValue::Integer(x)) => x as f64,
Some(OwnedValue::Float(x)) => x,
Some(OwnedValue::Null) => return OwnedValue::Null,
@ -3259,7 +3358,9 @@ fn exec_trim(reg: &OwnedValue, pattern: Option<OwnedValue>) -> OwnedValue {
}
_ => reg.to_owned(),
},
(OwnedValue::Text(t), None) => OwnedValue::build_text(Rc::new(t.value.trim().to_string())),
(OwnedValue::Text(t), None) => {
OwnedValue::build_text(Rc::new(t.as_str().trim().to_string()))
}
(reg, _) => reg.to_owned(),
}
}
@ -3279,7 +3380,7 @@ fn exec_ltrim(reg: &OwnedValue, pattern: Option<OwnedValue>) -> OwnedValue {
_ => reg.to_owned(),
},
(OwnedValue::Text(t), None) => {
OwnedValue::build_text(Rc::new(t.value.trim_start().to_string()))
OwnedValue::build_text(Rc::new(t.as_str().trim_start().to_string()))
}
(reg, _) => reg.to_owned(),
}
@ -3300,7 +3401,7 @@ fn exec_rtrim(reg: &OwnedValue, pattern: Option<OwnedValue>) -> OwnedValue {
_ => reg.to_owned(),
},
(OwnedValue::Text(t), None) => {
OwnedValue::build_text(Rc::new(t.value.trim_end().to_string()))
OwnedValue::build_text(Rc::new(t.as_str().trim_end().to_string()))
}
(reg, _) => reg.to_owned(),
}
@ -3310,7 +3411,7 @@ fn exec_zeroblob(req: &OwnedValue) -> OwnedValue {
let length: i64 = match req {
OwnedValue::Integer(i) => *i,
OwnedValue::Float(f) => *f as i64,
OwnedValue::Text(s) => s.value.parse().unwrap_or(0),
OwnedValue::Text(s) => s.as_str().parse().unwrap_or(0),
_ => 0,
};
OwnedValue::Blob(Rc::new(vec![0; length.max(0) as usize]))
@ -3352,7 +3453,7 @@ fn exec_cast(value: &OwnedValue, datatype: &str) -> OwnedValue {
let text = String::from_utf8_lossy(b);
cast_text_to_real(&text)
}
OwnedValue::Text(t) => cast_text_to_real(&t.value),
OwnedValue::Text(t) => cast_text_to_real(t.as_str()),
OwnedValue::Integer(i) => OwnedValue::Float(*i as f64),
OwnedValue::Float(f) => OwnedValue::Float(*f),
_ => OwnedValue::Float(0.0),
@ -3363,7 +3464,7 @@ fn exec_cast(value: &OwnedValue, datatype: &str) -> OwnedValue {
let text = String::from_utf8_lossy(b);
cast_text_to_integer(&text)
}
OwnedValue::Text(t) => cast_text_to_integer(&t.value),
OwnedValue::Text(t) => cast_text_to_integer(t.as_str()),
OwnedValue::Integer(i) => OwnedValue::Integer(*i),
// A cast of a REAL value into an INTEGER results in the integer between the REAL value and zero
// that is closest to the REAL value. If a REAL is greater than the greatest possible signed integer (+9223372036854775807)
@ -3386,7 +3487,7 @@ fn exec_cast(value: &OwnedValue, datatype: &str) -> OwnedValue {
let text = String::from_utf8_lossy(b);
cast_text_to_numeric(&text)
}
OwnedValue::Text(t) => cast_text_to_numeric(&t.value),
OwnedValue::Text(t) => cast_text_to_numeric(t.as_str()),
OwnedValue::Integer(i) => OwnedValue::Integer(*i),
OwnedValue::Float(f) => OwnedValue::Float(*f),
_ => value.clone(), // TODO probably wrong
@ -3414,13 +3515,13 @@ fn exec_replace(source: &OwnedValue, pattern: &OwnedValue, replacement: &OwnedVa
// If any of the casts failed, panic as text casting is not expected to fail.
match (&source, &pattern, &replacement) {
(OwnedValue::Text(source), OwnedValue::Text(pattern), OwnedValue::Text(replacement)) => {
if pattern.value.is_empty() {
return OwnedValue::build_text(source.value.clone());
if pattern.as_str().is_empty() {
return OwnedValue::Text(source.clone());
}
let result = source
.value
.replace(pattern.value.as_str(), &replacement.value);
.as_str()
.replace(pattern.as_str(), replacement.as_str());
OwnedValue::build_text(Rc::new(result))
}
_ => unreachable!("text cast should never fail"),
@ -3567,7 +3668,7 @@ fn to_f64(reg: &OwnedValue) -> Option<f64> {
match reg {
OwnedValue::Integer(i) => Some(*i as f64),
OwnedValue::Float(f) => Some(*f),
OwnedValue::Text(t) => t.value.parse::<f64>().ok(),
OwnedValue::Text(t) => t.as_str().parse::<f64>().ok(),
OwnedValue::Agg(ctx) => to_f64(ctx.final_value()),
_ => None,
}
@ -3934,7 +4035,7 @@ mod tests {
#[test]
fn test_unhex() {
let input = OwnedValue::build_text(Rc::new(String::from("6F")));
let input = OwnedValue::build_text(Rc::new(String::from("6f")));
let expected = OwnedValue::Blob(Rc::new(vec![0x6f]));
assert_eq!(exec_unhex(&input, None), expected);

12
core/vdbe/sorter.rs
View file

@ -1,9 +1,9 @@
use crate::types::OwnedRecord;
use crate::types::Record;
use std::cmp::Ordering;
pub struct Sorter {
records: Vec<OwnedRecord>,
current: Option<OwnedRecord>,
records: Vec<Record>,
current: Option<Record>,
order: Vec<bool>,
}
@ -51,11 +51,11 @@ impl Sorter {
pub fn next(&mut self) {
self.current = self.records.pop();
}
pub fn record(&self) -> Option<&OwnedRecord> {
pub fn record(&self) -> Option<&Record> {
self.current.as_ref()
}
pub fn insert(&mut self, record: &OwnedRecord) {
self.records.push(OwnedRecord::new(record.values.to_vec()));
pub fn insert(&mut self, record: &Record) {
self.records.push(Record::new(record.values.to_vec()));
}
}

119
extensions/core/README.md
View file

@ -9,7 +9,8 @@ like traditional `sqlite3` extensions, but are able to be written in much more e
- [ x ] **Scalar Functions**: Create scalar functions using the `scalar` macro.
- [ x ] **Aggregate Functions**: Define aggregate functions with `AggregateDerive` macro and `AggFunc` trait.
- [] **Virtual tables**: TODO
- [ x ] **Virtual tables**: Create a module for a virtual table with the `VTabModuleDerive` macro and `VTabCursor` trait.
- [] **VFS Modules**
---
## Installation
@ -17,24 +18,32 @@ like traditional `sqlite3` extensions, but are able to be written in much more e
Add the crate to your `Cargo.toml`:
```toml
[features]
static = ["limbo_ext/static"]
[dependencies]
limbo_ext = { path = "path/to/limbo/extensions/core" } # temporary until crate is published
limbo_ext = { path = "path/to/limbo/extensions/core", features = ["static"] } # temporary until crate is published
# mimalloc is required if you intend on linking dynamically. It is imported for you by the register_extension
# macro, so no configuration is needed. But it must be added to your Cargo.toml
[target.'cfg(not(target_family = "wasm"))'.dependencies]
mimalloc = { version = "*", default-features = false }
```
**NOTE** Crate must be of type `cdylib` if you wish to link dynamically
```
# NOTE: Crate must be of type `cdylib` if you wish to link dynamically
[lib]
crate-type = ["cdylib", "lib"]
```
`cargo build` will output a shared library that can be loaded with `.load target/debug/libyour_crate_name`
`cargo build` will output a shared library that can be loaded by the following options:
#### **CLI:**
`.load target/debug/libyour_crate_name`
#### **SQL:**
`SELECT load_extension('target/debug/libyour_crate_name')`
Extensions can be registered with the `register_extension!` macro:
@ -44,6 +53,7 @@ Extensions can be registered with the `register_extension!` macro:
register_extension!{
scalars: { double }, // name of your function, if different from attribute name
aggregates: { Percentile },
vtabs: { CsvVTable },
}
```
@ -140,4 +150,101 @@ impl AggFunc for Percentile {
}
```
### Virtual Table Example:
```rust
/// Example: A virtual table that operates on a CSV file as a database table.
/// This example assumes that the CSV file is located at "data.csv" in the current directory.
#[derive(Debug, VTabModuleDerive)]
struct CsvVTable;
impl VTabModule for CsvVTable {
type VCursor = CsvCursor;
/// Declare the name for your virtual table
const NAME: &'static str = "csv_data";
/// Declare the table schema and call `api.declare_virtual_table` with the schema sql.
fn connect(api: &ExtensionApi) -> ResultCode {
let sql = "CREATE TABLE csv_data(
name TEXT,
age TEXT,
city TEXT
)";
api.declare_virtual_table(Self::NAME, sql)
}
/// Open to return a new cursor: In this simple example, the CSV file is read completely into memory on connect.
fn open() -> Self::VCursor {
// Read CSV file contents from "data.csv"
let csv_content = fs::read_to_string("data.csv").unwrap_or_default();
// For simplicity, we'll ignore the header row.
let rows: Vec<Vec<String>> = csv_content
.lines()
.skip(1)
.map(|line| {
line.split(',')
.map(|s| s.trim().to_string())
.collect()
})
.collect();
CsvCursor { rows, index: 0 }
}
/// Filter through result columns. (not used in this simple example)
fn filter(_cursor: &mut Self::VCursor, _arg_count: i32, _args: &[Value]) -> ResultCode {
ResultCode::OK
}
/// Return the value for the column at the given index in the current row.
fn column(cursor: &Self::VCursor, idx: u32) -> Value {
cursor.column(idx)
}
/// Next advances the cursor to the next row.
fn next(cursor: &mut Self::VCursor) -> ResultCode {
if cursor.index < cursor.rows.len() - 1 {
cursor.index += 1;
ResultCode::OK
} else {
ResultCode::EOF
}
}
/// Return true if the cursor is at the end.
fn eof(cursor: &Self::VCursor) -> bool {
cursor.index >= cursor.rows.len()
}
}
/// The cursor for iterating over CSV rows.
#[derive(Debug)]
struct CsvCursor {
rows: Vec<Vec<String>>,
index: usize,
}
/// Implement the VTabCursor trait for your cursor type
impl VTabCursor for CsvCursor {
fn next(&mut self) -> ResultCode {
CsvCursor::next(self)
}
fn eof(&self) -> bool {
self.index >= self.rows.len()
}
fn column(&self, idx: u32) -> Value {
let row = &self.rows[self.index];
if (idx as usize) < row.len() {
Value::from_text(&row[idx as usize])
} else {
Value::null()
}
}
fn rowid(&self) -> i64 {
self.index as i64
}
}
```

76
extensions/core/src/lib.rs
View file

@ -1,5 +1,5 @@
mod types;
pub use limbo_macros::{register_extension, scalar, AggregateDerive};
pub use limbo_macros::{register_extension, scalar, AggregateDerive, VTabModuleDerive};
use std::os::raw::{c_char, c_void};
pub use types::{ResultCode, Value, ValueType};
@ -21,6 +21,30 @@ pub struct ExtensionApi {
step_func: StepFunction,
finalize_func: FinalizeFunction,
) -> ResultCode,
pub register_module: unsafe extern "C" fn(
ctx: *mut c_void,
name: *const c_char,
module: VTabModuleImpl,
) -> ResultCode,
pub declare_vtab: unsafe extern "C" fn(
ctx: *mut c_void,
name: *const c_char,
sql: *const c_char,
) -> ResultCode,
}
impl ExtensionApi {
pub fn declare_virtual_table(&self, name: &str, sql: &str) -> ResultCode {
let Ok(name) = std::ffi::CString::new(name) else {
return ResultCode::Error;
};
let Ok(sql) = std::ffi::CString::new(sql) else {
return ResultCode::Error;
};
unsafe { (self.declare_vtab)(self.ctx, name.as_ptr(), sql.as_ptr()) }
}
}
pub type ExtensionEntryPoint = unsafe extern "C" fn(api: *const ExtensionApi) -> ResultCode;
@ -47,3 +71,53 @@ pub trait AggFunc {
fn step(state: &mut Self::State, args: &[Value]);
fn finalize(state: Self::State) -> Value;
}
#[repr(C)]
#[derive(Clone, Debug)]
pub struct VTabModuleImpl {
pub name: *const c_char,
pub connect: VtabFnConnect,
pub open: VtabFnOpen,
pub filter: VtabFnFilter,
pub column: VtabFnColumn,
pub next: VtabFnNext,
pub eof: VtabFnEof,
}
pub type VtabFnConnect = unsafe extern "C" fn(api: *const c_void) -> ResultCode;
pub type VtabFnOpen = unsafe extern "C" fn() -> *mut c_void;
pub type VtabFnFilter =
unsafe extern "C" fn(cursor: *mut c_void, argc: i32, argv: *const Value) -> ResultCode;
pub type VtabFnColumn = unsafe extern "C" fn(cursor: *mut c_void, idx: u32) -> Value;
pub type VtabFnNext = unsafe extern "C" fn(cursor: *mut c_void) -> ResultCode;
pub type VtabFnEof = unsafe extern "C" fn(cursor: *mut c_void) -> bool;
pub trait VTabModule: 'static {
type VCursor: VTabCursor;
const NAME: &'static str;
fn connect(api: &ExtensionApi) -> ResultCode;
fn open() -> Self::VCursor;
fn filter(cursor: &mut Self::VCursor, arg_count: i32, args: &[Value]) -> ResultCode;
fn column(cursor: &Self::VCursor, idx: u32) -> Value;
fn next(cursor: &mut Self::VCursor) -> ResultCode;
fn eof(cursor: &Self::VCursor) -> bool;
}
pub trait VTabCursor: Sized {
type Error: std::fmt::Display;
fn rowid(&self) -> i64;
fn column(&self, idx: u32) -> Value;
fn eof(&self) -> bool;
fn next(&mut self) -> ResultCode;
}
#[repr(C)]
pub struct VTabImpl {
pub module: VTabModuleImpl,
}

34
extensions/core/src/types.rs
View file

@ -2,8 +2,8 @@ use std::fmt::Display;
/// Error type is of type ExtError which can be
/// either a user defined error or an error code
#[derive(Clone, Copy)]
#[repr(C)]
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum ResultCode {
OK = 0,
Error = 1,
@ -20,6 +20,7 @@ pub enum ResultCode {
Internal = 12,
Unavailable = 13,
CustomError = 14,
EOF = 15,
}
impl ResultCode {
@ -50,6 +51,7 @@ impl Display for ResultCode {
ResultCode::Internal => write!(f, "Internal Error"),
ResultCode::Unavailable => write!(f, "Unavailable"),
ResultCode::CustomError => write!(f, "Error "),
ResultCode::EOF => write!(f, "EOF"),
}
}
}
@ -204,6 +206,13 @@ impl Blob {
pub fn new(data: *const u8, size: u64) -> Self {
Self { data, size }
}
pub fn as_bytes(&self) -> &[u8] {
if self.data.is_null() {
return &[];
}
unsafe { std::slice::from_raw_parts(self.data, self.size as usize) }
}
}
impl Value {
@ -303,6 +312,29 @@ impl Value {
}
}
// Return ValueData as raw bytes
pub fn as_bytes(&self) -> Vec<u8> {
let mut bytes = vec![];
unsafe {
match self.value_type {
ValueType::Integer => bytes.extend_from_slice(&self.value.int.to_le_bytes()),
ValueType::Float => bytes.extend_from_slice(&self.value.float.to_le_bytes()),
ValueType::Text => {
let text = self.value.text.as_ref().expect("Invalid text pointer");
bytes.extend_from_slice(text.as_str().as_bytes());
}
ValueType::Blob => {
let blob = self.value.blob.as_ref().expect("Invalid blob pointer");
bytes.extend_from_slice(blob.as_bytes());
}
ValueType::Error | ValueType::Null => {}
}
}
bytes
}
/// Creates a new integer Value from an i64
pub fn from_integer(i: i64) -> Self {
Self {

24
extensions/crypto/Cargo.toml Normal file
View file

@ -0,0 +1,24 @@
[package]
name = "limbo_crypto"
version.workspace = true
authors.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
[lib]
crate-type = ["cdylib", "lib"]
[features]
static= [ "limbo_ext/static" ]
[dependencies]
blake3 = "1.5.5"
data-encoding = "2.7.0"
limbo_ext = { path = "../core", features = ["static"] }
md5 = "0.7.0"
ring = "0.17.8"
urlencoding = "2.1.3"
[target.'cfg(not(target_family = "wasm"))'.dependencies]
mimalloc = { version = "*", default-features = false }

215
extensions/crypto/src/crypto.rs Normal file
View file

@ -0,0 +1,215 @@
use crate::Error;
use blake3::Hasher;
use data_encoding::{BASE32, BASE64, HEXLOWER};
use limbo_ext::{Value, ValueType};
use ring::digest::{self, digest};
use std::{borrow::Cow, error::Error as StdError};
pub fn sha256(data: &Value) -> Result<Vec<u8>, Error> {
match data.value_type() {
ValueType::Error | ValueType::Null => Err(Error::InvalidType),
_ => {
let hash = digest(&digest::SHA256, &data.as_bytes());
Ok(hash.as_ref().to_vec())
}
}
}
pub fn sha512(data: &Value) -> Result<Vec<u8>, Error> {
match data.value_type() {
ValueType::Error | ValueType::Null => Err(Error::InvalidType),
_ => {
let hash = digest(&digest::SHA512, &data.as_bytes());
Ok(hash.as_ref().to_vec())
}
}
}
pub fn sha384(data: &Value) -> Result<Vec<u8>, Error> {
match data.value_type() {
ValueType::Error | ValueType::Null => Err(Error::InvalidType),
_ => {
let hash = digest(&digest::SHA384, &data.as_bytes());
Ok(hash.as_ref().to_vec())
}
}
}
pub fn blake3(data: &Value) -> Result<Vec<u8>, Error> {
match data.value_type() {
ValueType::Error | ValueType::Null => Err(Error::InvalidType),
_ => {
let mut hasher = Hasher::new();
hasher.update(data.as_bytes().as_ref());
Ok(hasher.finalize().as_bytes().to_vec())
}
}
}
pub fn sha1(data: &Value) -> Result<Vec<u8>, Error> {
match data.value_type() {
ValueType::Error | ValueType::Null => Err(Error::InvalidType),
_ => {
let hash = digest(&digest::SHA1_FOR_LEGACY_USE_ONLY, &data.as_bytes());
Ok(hash.as_ref().to_vec())
}
}
}
pub fn md5(data: &Value) -> Result<Vec<u8>, Error> {
match data.value_type() {
ValueType::Error | ValueType::Null => Err(Error::InvalidType),
_ => {
let digest = md5::compute::<&Vec<u8>>(data.as_bytes().as_ref());
Ok(digest.as_ref().to_vec())
}
}
}
pub fn encode(data: &Value, format: &Value) -> Result<Value, Error> {
match (data.value_type(), format.value_type()) {
(ValueType::Error, _) | (ValueType::Null, _) => Err(Error::InvalidType),
(_, ValueType::Text) => match format.to_text().unwrap().to_lowercase().as_str() {
"base32" => Ok(Value::from_text(BASE32.encode(data.as_bytes().as_ref()))),
"base64" => Ok(Value::from_text(BASE64.encode(data.as_bytes().as_ref()))),
"hex" => Ok(Value::from_text(HEXLOWER.encode(data.as_bytes().as_ref()))),
"base85" => Ok(Value::from_text(encode_ascii85(data.as_bytes().as_ref()))),
"url" => {
let data = data.as_bytes();
let url = urlencoding::encode_binary(&data);
Ok(Value::from_text(url.to_string()))
}
_ => Err(Error::UnknownOperation),
},
_ => Err(Error::InvalidType),
}
}
pub fn decode(data: &Value, format: &Value) -> Result<Value, Error> {
match (data.value_type(), format.value_type()) {
(ValueType::Error, _) | (ValueType::Null, _) => Err(Error::InvalidType),
(ValueType::Text, ValueType::Text) => {
let format_str = format.to_text().ok_or(Error::InvalidType)?.to_lowercase();
let input_text = data.to_text().ok_or(Error::InvalidType)?;
match format_str.as_str() {
"base32" => {
let payload = BASE32
.decode(input_text.as_bytes())
.map_err(|_| Error::DecodeFailed)?;
Ok(Value::from_text(
String::from_utf8(payload).map_err(|_| Error::InvalidUtf8)?,
))
}
"base64" => {
let payload = BASE64
.decode(input_text.as_bytes())
.map_err(|_| Error::DecodeFailed)?;
Ok(Value::from_text(
String::from_utf8(payload).map_err(|_| Error::InvalidUtf8)?,
))
}
"hex" => {
let payload = HEXLOWER
.decode(input_text.to_lowercase().as_bytes())
.map_err(|_| Error::DecodeFailed)?;
Ok(Value::from_text(
String::from_utf8(payload).map_err(|_| Error::InvalidUtf8)?,
))
}
"base85" => {
let decoded = decode_ascii85(&input_text).map_err(|_| Error::DecodeFailed)?;
Ok(Value::from_text(
String::from_utf8(decoded).map_err(|_| Error::InvalidUtf8)?,
))
}
"url" => {
let decoded = urlencoding::decode_binary(input_text.as_bytes());
Ok(Value::from_text(
String::from_utf8(decoded.to_vec()).map_err(|_| Error::InvalidUtf8)?,
))
}
_ => Err(Error::UnknownOperation),
}
}
_ => Err(Error::InvalidType),
}
}
// Ascii85 functions to avoid +1 dependency and to remove '~>' '<~'
const TABLE: [u32; 5] = [85 * 85 * 85 * 85, 85 * 85 * 85, 85 * 85, 85, 1];
fn decode_ascii85(input: &str) -> Result<Vec<u8>, Box<dyn StdError>> {
let mut result = Vec::with_capacity(4 * (input.len() / 5 + 16));
let mut counter = 0;
let mut chunk = 0;
for digit in input.trim().bytes().filter(|c| !c.is_ascii_whitespace()) {
if digit == b'z' {
if counter == 0 {
result.extend_from_slice(&[0, 0, 0, 0]);
} else {
return Err("Missaligned z in input".into());
}
}
if digit < 33 || digit > 117 {
return Err("Input char is out of range for Ascii85".into());
}
decode_digit(digit, &mut counter, &mut chunk, &mut result);
}
let mut to_remove = 0;
while counter != 0 {
decode_digit(b'u', &mut counter, &mut chunk, &mut result);
to_remove += 1;
}
result.drain((result.len() - to_remove)..result.len());
Ok(result)
}
fn decode_digit(digit: u8, counter: &mut usize, chunk: &mut u32, result: &mut Vec<u8>) {
let byte = digit - 33;
*chunk += byte as u32 * TABLE[*counter];
if *counter == 4 {
result.extend_from_slice(&chunk.to_be_bytes());
*chunk = 0;
*counter = 0;
} else {
*counter += 1;
}
}
fn encode_ascii85(input: &[u8]) -> String {
let mut result = String::with_capacity(5 * (input.len() / 4 + 16));
for chunk in input.chunks(4) {
let (chunk, count) = if chunk.len() == 4 {
(Cow::from(chunk), 5)
} else {
let mut new_chunk = Vec::new();
new_chunk.resize_with(4, || 0);
new_chunk[..chunk.len()].copy_from_slice(chunk);
(Cow::from(new_chunk), 5 - (4 - chunk.len()))
};
let number = u32::from_be_bytes(chunk.as_ref().try_into().expect("Internal Error"));
for i in 0..count {
let digit = (((number / TABLE[i]) % 85) + 33) as u8;
result.push(digit as char);
}
}
result
}

120
extensions/crypto/src/lib.rs Normal file
View file

@ -0,0 +1,120 @@
use crypto::{blake3, decode, encode, md5, sha1, sha256, sha384, sha512};
use limbo_ext::{register_extension, scalar, ResultCode, Value};
mod crypto;
#[derive(Debug)]
enum Error {
InvalidType,
UnknownOperation,
DecodeFailed,
InvalidUtf8,
}
#[scalar(name = "crypto_sha256", alias = "crypto_sha256")]
fn crypto_sha256(args: &[Value]) -> Value {
if args.len() != 1 {
return Value::error(ResultCode::Error);
}
let Ok(hash) = sha256(&args[0]) else {
return Value::error(ResultCode::Error);
};
Value::from_blob(hash)
}
#[scalar(name = "crypto_sha512", alias = "crypto_sha512")]
fn crypto_sha512(args: &[Value]) -> Value {
if args.len() != 1 {
return Value::error(ResultCode::Error);
}
let Ok(hash) = sha512(&args[0]) else {
return Value::error(ResultCode::Error);
};
Value::from_blob(hash)
}
#[scalar(name = "crypto_sha384", alias = "crypto_sha384")]
fn crypto_sha384(args: &[Value]) -> Value {
if args.len() != 1 {
return Value::error(ResultCode::Error);
}
let Ok(hash) = sha384(&args[0]) else {
return Value::error(ResultCode::Error);
};
Value::from_blob(hash)
}
#[scalar(name = "crypto_blake3", alias = "crypto_blake3")]
fn crypto_blake3(args: &[Value]) -> Value {
if args.len() != 1 {
return Value::error(ResultCode::Error);
}
let Ok(hash) = blake3(&args[0]) else {
return Value::error(ResultCode::Error);
};
Value::from_blob(hash)
}
#[scalar(name = "crypto_sha1", alias = "crypto_sha1")]
fn crypto_sha1(args: &[Value]) -> Value {
if args.len() != 1 {
return Value::error(ResultCode::Error);
}
let Ok(hash) = sha1(&args[0]) else {
return Value::error(ResultCode::Error);
};
Value::from_blob(hash)
}
#[scalar(name = "crypto_md5", alias = "crypto_md5")]
fn crypto_md5(args: &[Value]) -> Value {
if args.len() != 1 {
return Value::error(ResultCode::Error);
}
let Ok(hash) = md5(&args[0]) else {
return Value::error(ResultCode::Error);
};
Value::from_blob(hash)
}
#[scalar(name = "crypto_encode", alias = "crypto_encode")]
fn crypto_encode(args: &[Value]) -> Value {
if args.len() != 2 {
return Value::error(ResultCode::Error);
}
let Ok(payload) = encode(&args[0], &args[1]) else {
return Value::error(ResultCode::Error);
};
payload
}
#[scalar(name = "crypto_decode", alias = "crypto_decode")]
fn crypto_decode(args: &[Value]) -> Value {
if args.len() != 2 {
return Value::error(ResultCode::Error);
}
let Ok(payload) = decode(&args[0], &args[1]) else {
return Value::error(ResultCode::Error);
};
payload
}
register_extension! {
scalars: { crypto_sha256, crypto_sha512, crypto_sha384, crypto_blake3, crypto_sha1, crypto_md5, crypto_encode, crypto_decode },
}

21
extensions/series/Cargo.toml Normal file
View file

@ -0,0 +1,21 @@
[package]
name = "limbo_series"
version.workspace = true
authors.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
[features]
static = ["limbo_ext/static"]
[lib]
crate-type = ["cdylib", "lib"]
[dependencies]
limbo_ext = { path = "../core", features = ["static"] }
log = "0.4.20"
[target.'cfg(not(target_family = "wasm"))'.dependencies]
mimalloc = { version = "*", default-features = false }

118
extensions/series/src/lib.rs Normal file
View file

@ -0,0 +1,118 @@
use limbo_ext::{
register_extension, ExtensionApi, ResultCode, VTabCursor, VTabModule, VTabModuleDerive, Value,
};
register_extension! {
vtabs: { GenerateSeriesVTab }
}
macro_rules! try_option {
($expr:expr, $err:expr) => {
match $expr {
Some(val) => val,
None => return $err,
}
};
}
/// A virtual table that generates a sequence of integers
#[derive(Debug, VTabModuleDerive)]
struct GenerateSeriesVTab;
impl VTabModule for GenerateSeriesVTab {
type VCursor = GenerateSeriesCursor;
const NAME: &'static str = "generate_series";
fn connect(api: &ExtensionApi) -> ResultCode {
// Create table schema
let sql = "CREATE TABLE generate_series(
value INTEGER,
start INTEGER HIDDEN,
stop INTEGER HIDDEN,
step INTEGER HIDDEN
)";
api.declare_virtual_table(Self::NAME, sql)
}
fn open() -> Self::VCursor {
GenerateSeriesCursor {
start: 0,
stop: 0,
step: 0,
current: 0,
}
}
fn filter(cursor: &mut Self::VCursor, arg_count: i32, args: &[Value]) -> ResultCode {
// args are the start, stop, and step
if arg_count == 0 || arg_count > 3 {
return ResultCode::InvalidArgs;
}
let start = try_option!(args[0].to_integer(), ResultCode::InvalidArgs);
let stop = try_option!(
args.get(1).map(|v| v.to_integer().unwrap_or(i64::MAX)),
ResultCode::InvalidArgs
);
let step = try_option!(
args.get(2).map(|v| v.to_integer().unwrap_or(1)),
ResultCode::InvalidArgs
);
cursor.start = start;
cursor.current = start;
cursor.step = step;
cursor.stop = stop;
ResultCode::OK
}
fn column(cursor: &Self::VCursor, idx: u32) -> Value {
cursor.column(idx)
}
fn next(cursor: &mut Self::VCursor) -> ResultCode {
GenerateSeriesCursor::next(cursor)
}
fn eof(cursor: &Self::VCursor) -> bool {
cursor.eof()
}
}
/// The cursor for iterating over the generated sequence
#[derive(Debug)]
struct GenerateSeriesCursor {
start: i64,
stop: i64,
step: i64,
current: i64,
}
impl VTabCursor for GenerateSeriesCursor {
type Error = ResultCode;
fn next(&mut self) -> ResultCode {
let next_val = self.current.saturating_add(self.step);
if (self.step > 0 && next_val > self.stop) || (self.step < 0 && next_val < self.stop) {
return ResultCode::EOF;
}
self.current = next_val;
ResultCode::OK
}
fn eof(&self) -> bool {
(self.step > 0 && self.current > self.stop) || (self.step < 0 && self.current < self.stop)
}
fn column(&self, idx: u32) -> Value {
match idx {
0 => Value::from_integer(self.current),
1 => Value::from_integer(self.start),
2 => Value::from_integer(self.stop),
3 => Value::from_integer(self.step),
_ => Value::null(),
}
}
fn rowid(&self) -> i64 {
((self.current - self.start) / self.step) + 1
}
}

22
macros/src/args.rs
View file

@ -6,31 +6,32 @@ use syn::{Ident, LitStr, Token};
pub(crate) struct RegisterExtensionInput {
pub aggregates: Vec<Ident>,
pub scalars: Vec<Ident>,
pub vtabs: Vec<Ident>,
}
impl syn::parse::Parse for RegisterExtensionInput {
fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
let mut aggregates = Vec::new();
let mut scalars = Vec::new();
let mut vtabs = Vec::new();
while !input.is_empty() {
if input.peek(syn::Ident) && input.peek2(Token![:]) {
let section_name: Ident = input.parse()?;
input.parse::<Token![:]>()?;
if section_name == "aggregates" || section_name == "scalars" {
let names = ["aggregates", "scalars", "vtabs"];
if names.contains(&section_name.to_string().as_str()) {
let content;
syn::braced!(content in input);
let parsed_items = Punctuated::<Ident, Token![,]>::parse_terminated(&content)?
.into_iter()
.collect();
if section_name == "aggregates" {
aggregates = parsed_items;
} else {
scalars = parsed_items;
}
match section_name.to_string().as_str() {
"aggregates" => aggregates = parsed_items,
"scalars" => scalars = parsed_items,
"vtabs" => vtabs = parsed_items,
_ => unreachable!(),
};
if input.peek(Token![,]) {
input.parse::<Token![,]>()?;
@ -39,13 +40,14 @@ impl syn::parse::Parse for RegisterExtensionInput {
return Err(syn::Error::new(section_name.span(), "Unknown section"));
}
} else {
return Err(input.error("Expected aggregates: or scalars: section"));
return Err(input.error("Expected aggregates:, scalars:, or vtabs: section"));
}
}
Ok(Self {
aggregates,
scalars,
vtabs,
})
}
}

229
macros/src/lib.rs
View file

@ -324,6 +324,201 @@ pub fn derive_agg_func(input: TokenStream) -> TokenStream {
TokenStream::from(expanded)
}
/// Macro to derive a VTabModule for your extension. This macro will generate
/// the necessary functions to register your module with core. You must implement
/// the VTabModule trait for your struct, and the VTabCursor trait for your cursor.
/// ```ignore
///#[derive(Debug, VTabModuleDerive)]
///struct CsvVTab;
///impl VTabModule for CsvVTab {
/// type VCursor = CsvCursor;
/// const NAME: &'static str = "csv_data";
///
/// /// Declare the schema for your virtual table
/// fn connect(api: &ExtensionApi) -> ResultCode {
/// let sql = "CREATE TABLE csv_data(
/// name TEXT,
/// age TEXT,
/// city TEXT
/// )";
/// api.declare_virtual_table(Self::NAME, sql)
/// }
/// /// Open the virtual table and return a cursor
/// fn open() -> Self::VCursor {
/// let csv_content = fs::read_to_string("data.csv").unwrap_or_default();
/// let rows: Vec<Vec<String>> = csv_content
/// .lines()
/// .skip(1)
/// .map(|line| {
/// line.split(',')
/// .map(|s| s.trim().to_string())
/// .collect()
/// })
/// .collect();
/// CsvCursor { rows, index: 0 }
/// }
/// /// Filter the virtual table based on arguments (omitted here for simplicity)
/// fn filter(_cursor: &mut Self::VCursor, _arg_count: i32, _args: &[Value]) -> ResultCode {
/// ResultCode::OK
/// }
/// /// Return the value for a given column index
/// fn column(cursor: &Self::VCursor, idx: u32) -> Value {
/// cursor.column(idx)
/// }
/// /// Move the cursor to the next row
/// fn next(cursor: &mut Self::VCursor) -> ResultCode {
/// if cursor.index < cursor.rows.len() - 1 {
/// cursor.index += 1;
/// ResultCode::OK
/// } else {
/// ResultCode::EOF
/// }
/// }
/// fn eof(cursor: &Self::VCursor) -> bool {
/// cursor.index >= cursor.rows.len()
/// }
/// #[derive(Debug)]
/// struct CsvCursor {
/// rows: Vec<Vec<String>>,
/// index: usize,
///
/// impl CsvCursor {
/// /// Returns the value for a given column index.
/// fn column(&self, idx: u32) -> Value {
/// let row = &self.rows[self.index];
/// if (idx as usize) < row.len() {
/// Value::from_text(&row[idx as usize])
/// } else {
/// Value::null()
/// }
/// }
/// // Implement the VTabCursor trait for your virtual cursor
/// impl VTabCursor for CsvCursor {
/// fn next(&mut self) -> ResultCode {
/// Self::next(self)
/// }
/// fn eof(&self) -> bool {
/// self.index >= self.rows.len()
/// }
/// fn column(&self, idx: u32) -> Value {
/// self.column(idx)
/// }
/// fn rowid(&self) -> i64 {
/// self.index as i64
/// }
///
#[proc_macro_derive(VTabModuleDerive)]
pub fn derive_vtab_module(input: TokenStream) -> TokenStream {
let ast = parse_macro_input!(input as DeriveInput);
let struct_name = &ast.ident;
let register_fn_name = format_ident!("register_{}", struct_name);
let connect_fn_name = format_ident!("connect_{}", struct_name);
let open_fn_name = format_ident!("open_{}", struct_name);
let filter_fn_name = format_ident!("filter_{}", struct_name);
let column_fn_name = format_ident!("column_{}", struct_name);
let next_fn_name = format_ident!("next_{}", struct_name);
let eof_fn_name = format_ident!("eof_{}", struct_name);
let expanded = quote! {
impl #struct_name {
#[no_mangle]
unsafe extern "C" fn #connect_fn_name(
db: *const ::std::ffi::c_void,
) -> ::limbo_ext::ResultCode {
if db.is_null() {
return ::limbo_ext::ResultCode::Error;
}
let api = unsafe { &*(db as *const ExtensionApi) };
<#struct_name as ::limbo_ext::VTabModule>::connect(api)
}
#[no_mangle]
unsafe extern "C" fn #open_fn_name(
) -> *mut ::std::ffi::c_void {
let cursor = <#struct_name as ::limbo_ext::VTabModule>::open();
Box::into_raw(Box::new(cursor)) as *mut ::std::ffi::c_void
}
#[no_mangle]
unsafe extern "C" fn #filter_fn_name(
cursor: *mut ::std::ffi::c_void,
argc: i32,
argv: *const ::limbo_ext::Value,
) -> ::limbo_ext::ResultCode {
if cursor.is_null() {
return ::limbo_ext::ResultCode::Error;
}
let cursor = unsafe { &mut *(cursor as *mut <#struct_name as ::limbo_ext::VTabModule>::VCursor) };
let args = std::slice::from_raw_parts(argv, argc as usize);
<#struct_name as ::limbo_ext::VTabModule>::filter(cursor, argc, args)
}
#[no_mangle]
unsafe extern "C" fn #column_fn_name(
cursor: *mut ::std::ffi::c_void,
idx: u32,
) -> ::limbo_ext::Value {
if cursor.is_null() {
return ::limbo_ext::Value::error(ResultCode::Error);
}
let cursor = unsafe { &mut *(cursor as *mut <#struct_name as ::limbo_ext::VTabModule>::VCursor) };
<#struct_name as ::limbo_ext::VTabModule>::column(cursor, idx)
}
#[no_mangle]
unsafe extern "C" fn #next_fn_name(
cursor: *mut ::std::ffi::c_void,
) -> ::limbo_ext::ResultCode {
if cursor.is_null() {
return ::limbo_ext::ResultCode::Error;
}
let cursor = unsafe { &mut *(cursor as *mut <#struct_name as ::limbo_ext::VTabModule>::VCursor) };
<#struct_name as ::limbo_ext::VTabModule>::next(cursor)
}
#[no_mangle]
unsafe extern "C" fn #eof_fn_name(
cursor: *mut ::std::ffi::c_void,
) -> bool {
if cursor.is_null() {
return true;
}
let cursor = unsafe { &mut *(cursor as *mut <#struct_name as ::limbo_ext::VTabModule>::VCursor) };
<#struct_name as ::limbo_ext::VTabModule>::eof(cursor)
}
#[no_mangle]
pub unsafe extern "C" fn #register_fn_name(
api: *const ::limbo_ext::ExtensionApi
) -> ::limbo_ext::ResultCode {
if api.is_null() {
return ::limbo_ext::ResultCode::Error;
}
let api = &*api;
let name = <#struct_name as ::limbo_ext::VTabModule>::NAME;
// name needs to be a c str FFI compatible, NOT CString
let name_c = std::ffi::CString::new(name).unwrap();
let module = ::limbo_ext::VTabModuleImpl {
name: name_c.as_ptr(),
connect: Self::#connect_fn_name,
open: Self::#open_fn_name,
filter: Self::#filter_fn_name,
column: Self::#column_fn_name,
next: Self::#next_fn_name,
eof: Self::#eof_fn_name,
};
(api.register_module)(api.ctx, name_c.as_ptr(), module)
}
}
};
TokenStream::from(expanded)
}
/// Register your extension with 'core' by providing the relevant functions
///```ignore
///use limbo_ext::{register_extension, scalar, Value, AggregateDerive, AggFunc};
@ -362,6 +557,7 @@ pub fn register_extension(input: TokenStream) -> TokenStream {
let RegisterExtensionInput {
aggregates,
scalars,
vtabs,
} = input_ast;
let scalar_calls = scalars.iter().map(|scalar_ident| {
@ -388,8 +584,23 @@ pub fn register_extension(input: TokenStream) -> TokenStream {
}
}
});
let vtab_calls = vtabs.iter().map(|vtab_ident| {
let register_fn = syn::Ident::new(&format!("register_{}", vtab_ident), vtab_ident.span());
quote! {
{
let result = unsafe{ #vtab_ident::#register_fn(api)};
if result == ::limbo_ext::ResultCode::OK {
let result = <#vtab_ident as ::limbo_ext::VTabModule>::connect(api);
if !result.is_ok() {
return result;
}
}
}
}
});
let static_aggregates = aggregate_calls.clone();
let static_scalars = scalar_calls.clone();
let static_vtabs = vtab_calls.clone();
let expanded = quote! {
#[cfg(not(target_family = "wasm"))]
@ -404,19 +615,23 @@ pub fn register_extension(input: TokenStream) -> TokenStream {
#(#static_aggregates)*
#(#static_vtabs)*
::limbo_ext::ResultCode::OK
}
#[cfg(not(feature = "static"))]
#[no_mangle]
pub unsafe extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode {
let api = unsafe { &*api };
#(#scalar_calls)*
#[no_mangle]
pub unsafe extern "C" fn register_extension(api: &::limbo_ext::ExtensionApi) -> ::limbo_ext::ResultCode {
let api = unsafe { &*api };
#(#scalar_calls)*
#(#aggregate_calls)*
#(#aggregate_calls)*
::limbo_ext::ResultCode::OK
}
#(#vtab_calls)*
::limbo_ext::ResultCode::OK
}
};
TokenStream::from(expanded)

206
testing/extensions.py
View file

@ -115,10 +115,14 @@ def validate_string_uuid(result):
return len(result) == 36 and result.count("-") == 4
def returns_error(result):
def returns_error_no_func(result):
return "error: no such function: " in result
def returns_vtable_parse_err(result):
return "Parse error: Virtual table" in result
def returns_null(result):
return result == "" or result == "\n"
@ -167,7 +171,7 @@ def test_regexp(pipe):
extension_path = "./target/debug/liblimbo_regexp.so"
# before extension loads, assert no function
run_test(pipe, "SELECT regexp('a.c', 'abc');", returns_error)
run_test(pipe, "SELECT regexp('a.c', 'abc');", returns_error_no_func)
run_test(pipe, f".load {extension_path}", returns_null)
print(f"Extension {extension_path} loaded successfully.")
run_test(pipe, "SELECT regexp('a.c', 'abc');", validate_true)
@ -214,7 +218,7 @@ def test_aggregates(pipe):
run_test(
pipe,
"SELECT median(1);",
returns_error,
returns_error_no_func,
"median agg function returns null when ext not loaded",
)
run_test(
@ -256,12 +260,208 @@ def test_aggregates(pipe):
)
# Encoders and decoders
def validate_url_encode(a):
return a == "%2Fhello%3Ftext%3D%28%E0%B2%A0_%E0%B2%A0%29"
def validate_url_decode(a):
return a == "/hello?text=(ಠ_ಠ)"
def validate_hex_encode(a):
return a == "68656c6c6f"
def validate_hex_decode(a):
return a == "hello"
def validate_base85_encode(a):
return a == "BOu!rDZ"
def validate_base85_decode(a):
return a == "hello"
def validate_base32_encode(a):
return a == "NBSWY3DP"
def validate_base32_decode(a):
return a == "hello"
def validate_base64_encode(a):
return a == "aGVsbG8="
def validate_base64_decode(a):
return a == "hello"
def test_crypto(pipe):
extension_path = "./target/debug/liblimbo_crypto.so"
# assert no function before extension loads
run_test(
pipe,
"SELECT crypto_blake('a');",
lambda res: "Parse error" in res,
"crypto_blake3 returns null when ext not loaded",
)
run_test(
pipe,
f".load {extension_path}",
returns_null,
"load extension command works properly",
)
# Hashing and Decode
run_test(
pipe,
"SELECT crypto_encode(crypto_blake3('abc'), 'hex');",
lambda res: res
== "6437b3ac38465133ffb63b75273a8db548c558465d79db03fd359c6cd5bd9d85",
"blake3 should encrypt correctly",
)
run_test(
pipe,
"SELECT crypto_encode(crypto_md5('abc'), 'hex');",
lambda res: res == "900150983cd24fb0d6963f7d28e17f72",
"md5 should encrypt correctly",
)
run_test(
pipe,
"SELECT crypto_encode(crypto_sha1('abc'), 'hex');",
lambda res: res == "a9993e364706816aba3e25717850c26c9cd0d89d",
"sha1 should encrypt correctly",
)
run_test(
pipe,
"SELECT crypto_encode(crypto_sha256('abc'), 'hex');",
lambda a: a
== "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
"sha256 should encrypt correctly",
)
run_test(
pipe,
"SELECT crypto_encode(crypto_sha384('abc'), 'hex');",
lambda a: a
== "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed8086072ba1e7cc2358baeca134c825a7",
"sha384 should encrypt correctly",
)
run_test(
pipe,
"SELECT crypto_encode(crypto_sha512('abc'), 'hex');",
lambda a: a
== "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f",
"sha512 should encrypt correctly",
)
# Encoding and Decoding
run_test(
pipe,
"SELECT crypto_encode('hello', 'base32');",
validate_base32_encode,
"base32 should encode correctly",
)
run_test(
pipe,
"SELECT crypto_decode('NBSWY3DP', 'base32');",
validate_base32_decode,
"base32 should decode correctly",
)
run_test(
pipe,
"SELECT crypto_encode('hello', 'base64');",
validate_base64_encode,
"base64 should encode correctly",
)
run_test(
pipe,
"SELECT crypto_decode('aGVsbG8=', 'base64');",
validate_base64_decode,
"base64 should decode correctly",
)
run_test(
pipe,
"SELECT crypto_encode('hello', 'base85');",
validate_base85_encode,
"base85 should encode correctly",
)
run_test(
pipe,
"SELECT crypto_decode('BOu!rDZ', 'base85');",
validate_base85_decode,
"base85 should decode correctly",
)
run_test(
pipe,
"SELECT crypto_encode('hello', 'hex');",
validate_hex_encode,
"hex should encode correctly",
)
run_test(
pipe,
"SELECT crypto_decode('68656c6c6f', 'hex');",
validate_hex_decode,
"hex should decode correctly",
)
run_test(
pipe,
"SELECT crypto_encode('/hello?text=(ಠ_ಠ)', 'url');",
validate_url_encode,
"url should encode correctly",
)
run_test(
pipe,
"SELECT crypto_decode('%2Fhello%3Ftext%3D%28%E0%B2%A0_%E0%B2%A0%29', 'url');",
validate_url_decode,
"url should decode correctly",
)
def test_series(pipe):
ext_path = "./target/debug/liblimbo_series"
run_test(
pipe,
"SELECT * FROM generate_series(1, 10);",
lambda res: "Virtual table generate_series not found" in res,
)
run_test(pipe, f".load {ext_path}", returns_null)
run_test(
pipe,
"SELECT * FROM generate_series(1, 10);",
lambda res: "Invalid Argument" in res,
)
run_test(
pipe,
"SELECT * FROM generate_series(1, 10, 2);",
lambda res: res == "1\n3\n5\n7\n9",
)
run_test(
pipe,
"SELECT * FROM generate_series(1, 10, 2, 3);",
lambda res: "Invalid Argument" in res,
)
run_test(
pipe,
"SELECT * FROM generate_series(10, 1, -2);",
lambda res: res == "10\n8\n6\n4\n2",
)
def main():
pipe = init_limbo()
try:
test_regexp(pipe)
test_uuid(pipe)
test_aggregates(pipe)
test_crypto(pipe)
test_series(pipe)
except Exception as e:
print(f"Test FAILED: {e}")
pipe.terminate()