mirrors/limbo
1
0
Fork 0
mirror of https://github.com/tursodatabase/limbo.git synced 2025-08-04 18:18:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

Merge branch 'main' into java-bindings-statement-refactor

Browse source
This commit is contained in:
김선우 2025-02-09 10:26:29 +09:00
commit 3920539c7e
25 changed files with 908 additions and 250 deletions
Download patch file Download diff file Expand all files Collapse all files

2
COMPAT.md
View file

@ -380,7 +380,7 @@ Modifiers:
| json_type(json,path) | Yes | |
| json_valid(json) | Yes | |
| json_valid(json,flags) | | |
| json_quote(value) | | |
| json_quote(value) | Yes | |
| json_group_array(value) | | |
| jsonb_group_array(value) | | |
| json_group_object(label,value) | | |

20
Cargo.lock generated
View file

@ -1394,15 +1394,6 @@ version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674"
[[package]]
name = "java-limbo"
version = "0.0.14"
dependencies = [
"jni",
"limbo_core",
"thiserror 2.0.11",
]
[[package]]
name = "jni"
version = "0.21.1"
@ -1590,6 +1581,15 @@ dependencies = [
"limbo_core",
]
[[package]]
name = "limbo-java"
version = "0.0.14"
dependencies = [
"jni",
"limbo_core",
"thiserror 2.0.11",
]
[[package]]
name = "limbo-wasm"
version = "0.0.14"
@ -1735,6 +1735,8 @@ dependencies = [
"notify",
"rand 0.8.5",
"rand_chacha 0.3.1",
"regex",
"regex-syntax",
"serde",
"serde_json",
"tempfile",

4
Makefile
View file

@ -89,3 +89,7 @@ test-time:
test-sqlite3: limbo-c
LIBS="$(SQLITE_LIB)" HEADERS="$(SQLITE_LIB_HEADERS)" make -C sqlite3/tests test
.PHONY: test-sqlite3
test-json:
SQLITE_EXEC=$(SQLITE_EXEC) ./testing/json.test
.PHONY: test-json

5
README.md
View file

@ -142,6 +142,11 @@ for rows.Next() {
}
```
### ☕️ Java (wip)
We integrated Limbo into JDBC. For detailed instructions on how to use Limbo with java, please refer to
the [README.md under bindings/java](bindings/java/README.md).
## Contributing
We'd love to have you contribute to Limbo! Please check out the [contribution guide] to get started.

2
bindings/java/Cargo.toml
View file

@ -1,5 +1,5 @@
[package]
name = "java-limbo"
name = "limbo-java"
version.workspace = true
authors.workspace = true
edition.workspace = true

3
bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ResultSetTest.java
View file

@ -16,7 +16,6 @@ import java.util.Properties;
import java.util.stream.Stream;
import org.github.tursodatabase.TestUtils;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@ -241,7 +240,6 @@ class JDBC4ResultSetTest {
}
@Test
@Disabled("limbo has a bug which sees -9223372036854775808 as double")
void test_getLong() throws Exception {
stmt.executeUpdate("CREATE TABLE test_long (long_col BIGINT);");
stmt.executeUpdate("INSERT INTO test_long (long_col) VALUES (1234567890);");
@ -253,6 +251,7 @@ class JDBC4ResultSetTest {
ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_long");
// Test typical long value
assertTrue(resultSet.next());
assertEquals(1234567890L, resultSet.getLong(1));
// Test maximum long value

53
cli/app.rs
View file

@ -2,7 +2,8 @@ use crate::{
import::{ImportFile, IMPORT_HELP},
opcodes_dictionary::OPCODE_DESCRIPTIONS,
};
use cli_table::{Cell, Table};
use cli_table::format::{Border, HorizontalLine, Separator, VerticalLine};
use cli_table::{Cell, Style, Table};
use limbo_core::{Database, LimboError, Statement, StepResult, Value};
use clap::{Parser, ValueEnum};
@ -670,6 +671,16 @@ impl Limbo {
return Ok(());
}
let mut table_rows: Vec<Vec<_>> = vec![];
if rows.num_columns() > 0 {
let columns = (0..rows.num_columns())
.map(|i| {
rows.get_column_name(i)
.map(|name| name.cell().bold(true))
.unwrap_or_else(|| " ".cell())
})
.collect::<Vec<_>>();
table_rows.push(columns);
}
loop {
match rows.step() {
Ok(StepResult::Row) => {
@ -707,11 +718,7 @@ impl Limbo {
}
}
}
if let Ok(table) = table_rows.table().display() {
let _ = self.write_fmt(format_args!("{}", table));
} else {
let _ = self.writeln("Error displaying table.");
}
self.print_table(table_rows);
}
},
Ok(None) => {}
@ -727,6 +734,40 @@ impl Limbo {
Ok(())
}
fn print_table(&mut self, table_rows: Vec<Vec<cli_table::CellStruct>>) {
if table_rows.is_empty() {
return;
}
let horizontal_line = HorizontalLine::new('┌', '┐', '┬', '─');
let horizontal_line_mid = HorizontalLine::new('├', '┤', '┼', '─');
let horizontal_line_bottom = HorizontalLine::new('└', '┘', '┴', '─');
let vertical_line = VerticalLine::new('│');
let border = Border::builder()
.top(horizontal_line)
.bottom(horizontal_line_bottom)
.left(vertical_line.clone())
.right(vertical_line.clone())
.build();
let separator = Separator::builder()
.column(Some(vertical_line))
.row(Some(horizontal_line_mid))
.build();
if let Ok(table) = table_rows
.table()
.border(border)
.separator(separator)
.display()
{
let _ = self.write_fmt(format_args!("{}", table));
} else {
let _ = self.writeln("Error displaying table.");
}
}
fn display_schema(&mut self, table: Option<&str>) -> anyhow::Result<()> {
let sql = match table {
Some(table_name) => format!(

4
core/function.rs
View file

@ -84,6 +84,7 @@ pub enum JsonFunc {
JsonRemove,
JsonPretty,
JsonSet,
JsonQuote,
}
#[cfg(feature = "json")]
@ -107,6 +108,7 @@ impl Display for JsonFunc {
Self::JsonRemove => "json_remove".to_string(),
Self::JsonPretty => "json_pretty".to_string(),
Self::JsonSet => "json_set".to_string(),
Self::JsonQuote => "json_quote".to_string(),
}
)
}
@ -568,6 +570,8 @@ impl Func {
"json_pretty" => Ok(Self::Json(JsonFunc::JsonPretty)),
#[cfg(feature = "json")]
"json_set" => Ok(Self::Json(JsonFunc::JsonSet)),
#[cfg(feature = "json")]
"json_quote" => Ok(Self::Json(JsonFunc::JsonQuote)),
"unixepoch" => Ok(Self::Scalar(ScalarFunc::UnixEpoch)),
"julianday" => Ok(Self::Scalar(ScalarFunc::JulianDay)),
"hex" => Ok(Self::Scalar(ScalarFunc::Hex)),

19
core/io/io_uring.rs
View file

@ -4,7 +4,6 @@ use log::{debug, trace};
use rustix::fs::{self, FlockOperation, OFlags};
use rustix::io_uring::iovec;
use std::cell::RefCell;
use std::collections::HashMap;
use std::fmt;
use std::io::ErrorKind;
use std::os::fd::AsFd;
@ -40,7 +39,7 @@ pub struct UringIO {
struct WrappedIOUring {
ring: io_uring::IoUring,
pending_ops: usize,
pub pending: HashMap<u64, Rc<Completion>>,
pub pending: [Option<Rc<Completion>>; MAX_IOVECS as usize + 1],
key: u64,
}
@ -63,7 +62,7 @@ impl UringIO {
ring: WrappedIOUring {
ring,
pending_ops: 0,
pending: HashMap::new(),
pending: [const { None }; MAX_IOVECS as usize + 1],
key: 0,
},
iovecs: [iovec {
@ -92,7 +91,7 @@ impl InnerUringIO {
impl WrappedIOUring {
fn submit_entry(&mut self, entry: &io_uring::squeue::Entry, c: Rc<Completion>) {
trace!("submit_entry({:?})", entry);
self.pending.insert(entry.get_user_data(), c);
self.pending[entry.get_user_data() as usize] = Some(c);
unsafe {
self.ring
.submission()
@ -124,6 +123,11 @@ impl WrappedIOUring {
fn get_key(&mut self) -> u64 {
self.key += 1;
if self.key == MAX_IOVECS as u64 {
let key = self.key;
self.key = 0;
return key;
}
self.key
}
}
@ -175,10 +179,11 @@ impl IO for UringIO {
)));
}
{
let c = ring.pending.get(&cqe.user_data()).unwrap().clone();
c.complete(cqe.result());
if let Some(c) = ring.pending[cqe.user_data() as usize].as_ref() {
c.complete(cqe.result());
}
}
ring.pending.remove(&cqe.user_data());
ring.pending[cqe.user_data() as usize] = None;
}
Ok(())
}

37
core/json/mod.rs
View file

@ -674,6 +674,43 @@ pub fn is_json_valid(json_value: &OwnedValue) -> crate::Result<OwnedValue> {
}
}
pub fn json_quote(value: &OwnedValue) -> crate::Result<OwnedValue> {
match value {
OwnedValue::Text(ref t) => {
// If X is a JSON value returned by another JSON function,
// then this function is a no-op
if t.subtype == TextSubtype::Json {
// Should just return the json value with no quotes
return Ok(value.to_owned());
}
let mut escaped_value = String::with_capacity(t.value.len() + 4);
escaped_value.push('"');
for c in t.as_str().chars() {
match c {
'"' | '\\' | '\n' | '\r' | '\t' | '\u{0008}' | '\u{000c}' => {
escaped_value.push('\\');
escaped_value.push(c);
}
c => escaped_value.push(c),
}
}
escaped_value.push('"');
Ok(OwnedValue::Text(Text::new(Rc::new(escaped_value))))
}
// Numbers are unquoted in json
OwnedValue::Integer(ref int) => Ok(OwnedValue::Integer(int.to_owned())),
OwnedValue::Float(ref float) => Ok(OwnedValue::Float(float.to_owned())),
OwnedValue::Blob(_) => crate::bail_constraint_error!("JSON cannot hold BLOB values"),
OwnedValue::Null => Ok(OwnedValue::Text(Text::new(Rc::new("null".to_string())))),
_ => {
unreachable!()
}
}
}
#[cfg(test)]
mod tests {
use super::*;

162
core/mvcc/database/mod.rs
View file

@ -58,7 +58,9 @@ impl<T> LogRecord<T> {
/// versions switch to tracking timestamps.
#[derive(Clone, Debug, PartialEq, PartialOrd)]
enum TxTimestampOrID {
/// A committed transaction's timestamp.
Timestamp(u64),
/// The ID of a non-committed transaction.
TxID(TxID),
}
@ -229,55 +231,6 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
}
}
// Extracts the begin timestamp from a transaction
fn get_begin_timestamp(&self, ts_or_id: &TxTimestampOrID) -> u64 {
match ts_or_id {
TxTimestampOrID::Timestamp(ts) => *ts,
TxTimestampOrID::TxID(tx_id) => {
self.txs
.get(tx_id)
.unwrap()
.value()
.read()
.unwrap()
.begin_ts
}
}
}
/// Inserts a new row version into the database, while making sure that
/// the row version is inserted in the correct order.
fn insert_version(&self, id: RowID, row_version: RowVersion<T>) {
let versions = self.rows.get_or_insert_with(id, || RwLock::new(Vec::new()));
let mut versions = versions.value().write().unwrap();
self.insert_version_raw(&mut versions, row_version)
}
/// Inserts a new row version into the internal data structure for versions,
/// while making sure that the row version is inserted in the correct order.
fn insert_version_raw(&self, versions: &mut Vec<RowVersion<T>>, row_version: RowVersion<T>) {
// NOTICE: this is an insert a'la insertion sort, with pessimistic linear complexity.
// However, we expect the number of versions to be nearly sorted, so we deem it worthy
// to search linearly for the insertion point instead of paying the price of using
// another data structure, e.g. a BTreeSet. If it proves to be too quadratic empirically,
// we can either switch to a tree-like structure, or at least use partition_point()
// which performs a binary search for the insertion point.
let position = versions
.iter()
.rposition(|v| {
self.get_begin_timestamp(&v.begin) < self.get_begin_timestamp(&row_version.begin)
})
.map(|p| p + 1)
.unwrap_or(0);
if versions.len() - position > 3 {
tracing::debug!(
"Inserting a row version {} positions from the end",
versions.len() - position
);
}
versions.insert(position, row_version);
}
/// Inserts a new row into the database.
///
/// This function inserts a new `row` into the database within the context
@ -365,6 +318,10 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
.ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
let tx = tx.value().read().unwrap();
assert_eq!(tx.state, TransactionState::Active);
let version_is_visible_to_current_tx = is_version_visible(&self.txs, &tx, rv);
if !version_is_visible_to_current_tx {
continue;
}
if is_write_write_conflict(&self.txs, &tx, rv) {
drop(row_versions);
drop(row_versions_opt);
@ -372,19 +329,18 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
self.rollback_tx(tx_id);
return Err(DatabaseError::WriteWriteConflict);
}
if is_version_visible(&self.txs, &tx, rv) {
rv.end = Some(TxTimestampOrID::TxID(tx.tx_id));
drop(row_versions);
drop(row_versions_opt);
drop(tx);
let tx = self
.txs
.get(&tx_id)
.ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
let mut tx = tx.value().write().unwrap();
tx.insert_to_write_set(id);
return Ok(true);
}
rv.end = Some(TxTimestampOrID::TxID(tx.tx_id));
drop(row_versions);
drop(row_versions_opt);
drop(tx);
let tx = self
.txs
.get(&tx_id)
.ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
let mut tx = tx.value().write().unwrap();
tx.insert_to_write_set(id);
return Ok(true);
}
}
Ok(false)
@ -556,7 +512,6 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
*/
tx.state.store(TransactionState::Committed(end_ts));
tracing::trace!("COMMIT {tx}");
let tx_begin_ts = tx.begin_ts;
let write_set: Vec<RowID> = tx.write_set.iter().map(|v| *v.value()).collect();
drop(tx);
// Postprocessing: inserting row versions and logging the transaction to persistent storage.
@ -568,7 +523,9 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
for row_version in row_versions.iter_mut() {
if let TxTimestampOrID::TxID(id) = row_version.begin {
if id == tx_id {
row_version.begin = TxTimestampOrID::Timestamp(tx_begin_ts);
// New version is valid STARTING FROM committing transaction's end timestamp
// See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf
row_version.begin = TxTimestampOrID::Timestamp(end_ts);
self.insert_version_raw(
&mut log_record.row_versions,
row_version.clone(),
@ -577,6 +534,8 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
}
if let Some(TxTimestampOrID::TxID(id)) = row_version.end {
if id == tx_id {
// New version is valid UNTIL committing transaction's end timestamp
// See diagram on page 299: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf
row_version.end = Some(TxTimestampOrID::Timestamp(end_ts));
self.insert_version_raw(
&mut log_record.row_versions,
@ -718,10 +677,69 @@ impl<Clock: LogicalClock, T: Sync + Send + Clone + Debug + 'static> MvStore<Cloc
}
Ok(())
}
// Extracts the begin timestamp from a transaction
fn get_begin_timestamp(&self, ts_or_id: &TxTimestampOrID) -> u64 {
match ts_or_id {
TxTimestampOrID::Timestamp(ts) => *ts,
TxTimestampOrID::TxID(tx_id) => {
self.txs
.get(tx_id)
.unwrap()
.value()
.read()
.unwrap()
.begin_ts
}
}
}
/// Inserts a new row version into the database, while making sure that
/// the row version is inserted in the correct order.
fn insert_version(&self, id: RowID, row_version: RowVersion<T>) {
let versions = self.rows.get_or_insert_with(id, || RwLock::new(Vec::new()));
let mut versions = versions.value().write().unwrap();
self.insert_version_raw(&mut versions, row_version)
}
/// Inserts a new row version into the internal data structure for versions,
/// while making sure that the row version is inserted in the correct order.
fn insert_version_raw(&self, versions: &mut Vec<RowVersion<T>>, row_version: RowVersion<T>) {
// NOTICE: this is an insert a'la insertion sort, with pessimistic linear complexity.
// However, we expect the number of versions to be nearly sorted, so we deem it worthy
// to search linearly for the insertion point instead of paying the price of using
// another data structure, e.g. a BTreeSet. If it proves to be too quadratic empirically,
// we can either switch to a tree-like structure, or at least use partition_point()
// which performs a binary search for the insertion point.
let position = versions
.iter()
.rposition(|v| {
self.get_begin_timestamp(&v.begin) < self.get_begin_timestamp(&row_version.begin)
})
.map(|p| p + 1)
.unwrap_or(0);
if versions.len() - position > 3 {
tracing::debug!(
"Inserting a row version {} positions from the end",
versions.len() - position
);
}
versions.insert(position, row_version);
}
}
/// A write-write conflict happens when transaction T_m attempts to update a
/// row version that is currently being updated by an active transaction T_n.
/// A write-write conflict happens when transaction T_current attempts to update a
/// row version that is:
/// a) currently being updated by an active transaction T_previous, or
/// b) was updated by an ended transaction T_previous that committed AFTER T_current started
/// but BEFORE T_previous commits.
///
/// "Suppose transaction T wants to update a version V. V is updatable
/// only if it is the latest version, that is, it has an end timestamp equal
/// to infinity or its End field contains the ID of a transaction TE and
/// TE’s state is Aborted"
/// Ref: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf , page 301,
/// 2.6. Updating a Version.
pub(crate) fn is_write_write_conflict<T>(
txs: &SkipMap<TxID, RwLock<Transaction>>,
tx: &Transaction,
@ -731,12 +749,16 @@ pub(crate) fn is_write_write_conflict<T>(
Some(TxTimestampOrID::TxID(rv_end)) => {
let te = txs.get(&rv_end).unwrap();
let te = te.value().read().unwrap();
match te.state.load() {
TransactionState::Active | TransactionState::Preparing => tx.tx_id != te.tx_id,
_ => false,
if te.tx_id == tx.tx_id {
return false;
}
te.state.load() != TransactionState::Aborted
}
Some(TxTimestampOrID::Timestamp(_)) => false,
// A non-"infinity" end timestamp (here modeled by Some(ts)) functions as a write lock
// on the row, so it can never be updated by another transaction.
// Ref: https://www.cs.cmu.edu/~15721-f24/papers/Hekaton.pdf , page 301,
// 2.6. Updating a Version.
Some(TxTimestampOrID::Timestamp(_)) => true,
None => false,
}
}

16
core/mvcc/database/tests.rs
View file

@ -382,7 +382,7 @@ fn test_fuzzy_read() {
table_id: 1,
row_id: 1,
},
data: "Hello".to_string(),
data: "First".to_string(),
};
db.insert(tx1, tx1_row.clone()).unwrap();
let row = db
@ -419,7 +419,7 @@ fn test_fuzzy_read() {
table_id: 1,
row_id: 1,
},
data: "World".to_string(),
data: "Second".to_string(),
};
db.update(tx3, tx3_row).unwrap();
db.commit_tx(tx3).unwrap();
@ -436,6 +436,18 @@ fn test_fuzzy_read() {
.unwrap()
.unwrap();
assert_eq!(tx1_row, row);
// T2 tries to update the row, but fails because T3 has already committed an update to the row,
// so T2 trying to write would violate snapshot isolation if it succeeded.
let tx2_newrow = Row {
id: RowID {
table_id: 1,
row_id: 1,
},
data: "Third".to_string(),
};
let update_result = db.update(tx2, tx2_newrow);
assert_eq!(Err(DatabaseError::WriteWriteConflict), update_result);
}
#[test]

13
core/translate/expr.rs
View file

@ -975,7 +975,7 @@ pub fn translate_expr(
translate_function(
program,
&args,
args,
referenced_tables,
resolver,
target_register,
@ -1017,6 +1017,17 @@ pub fn translate_expr(
});
Ok(target_register)
}
JsonFunc::JsonQuote => {
let args = expect_arguments_exact!(args, 1, j);
translate_function(
program,
args,
referenced_tables,
resolver,
target_register,
func_ctx,
)
}
JsonFunc::JsonPretty => {
let args = expect_arguments_max!(args, 2, j);

10
core/vdbe/mod.rs
View file

@ -49,7 +49,7 @@ use crate::{
function::JsonFunc, json::get_json, json::is_json_valid, json::json_array,
json::json_array_length, json::json_arrow_extract, json::json_arrow_shift_extract,
json::json_error_position, json::json_extract, json::json_object, json::json_patch,
json::json_remove, json::json_set, json::json_type,
json::json_quote, json::json_remove, json::json_set, json::json_type,
};
use crate::{resolve_ext_path, Connection, Result, TransactionState, DATABASE_VERSION};
use insn::{
@ -1973,6 +1973,14 @@ impl Program {
Err(e) => return Err(e),
}
}
JsonFunc::JsonQuote => {
let json_value = &state.registers[*start_reg];
match json_quote(json_value) {
Ok(result) => state.registers[*dest] = result,
Err(e) => return Err(e),
}
}
},
crate::function::Func::Scalar(scalar_func) => match scalar_func {
ScalarFunc::Cast => {

2
simulator/Cargo.toml
View file

@ -21,6 +21,8 @@ rand_chacha = "0.3.1"
log = "0.4.20"
tempfile = "3.0.7"
env_logger = "0.10.1"
regex = "1.11.1"
regex-syntax = { version = "0.8.5", default-features = false, features = ["unicode"] }
anarchist-readable-name-generator-lib = "0.1.2"
clap = { version = "4.5", features = ["derive"] }
serde = { version = "1.0", features = ["derive"] }

37
simulator/generation/mod.rs
View file

@ -25,6 +25,13 @@ pub trait ArbitraryFrom<T> {
fn arbitrary_from<R: Rng>(rng: &mut R, t: T) -> Self;
}
/// ArbitraryFromMaybe trait for fallibally generating random values from a given value
pub trait ArbitraryFromMaybe<T> {
fn arbitrary_from_maybe<R: Rng>(rng: &mut R, t: T) -> Option<Self>
where
Self: Sized;
}
/// Frequency is a helper function for composing different generators with different frequency
/// of occurences.
/// The type signature for the `N` parameter is a bit complex, but it
@ -60,6 +67,36 @@ pub(crate) fn one_of<'a, T, R: Rng>(choices: Vec<Box<dyn Fn(&mut R) -> T + 'a>>,
choices[index](rng)
}
/// backtrack is a helper function for composing different "failable" generators.
/// The function takes a list of functions that return an Option<T>, along with number of retries
/// to make before giving up.
pub(crate) fn backtrack<'a, T, R: Rng>(
mut choices: Vec<(u32, Box<dyn Fn(&mut R) -> Option<T> + 'a>)>,
rng: &mut R,
) -> T {
loop {
// If there are no more choices left, we give up
let choices_ = choices
.iter()
.enumerate()
.filter(|(_, (retries, _))| *retries > 0)
.collect::<Vec<_>>();
if choices_.is_empty() {
panic!("backtrack: no more choices left");
}
// Run a one_of on the remaining choices
let (choice_index, choice) = pick(&choices_, rng);
let choice_index = *choice_index;
// If the choice returns None, we decrement the number of retries and try again
let result = choice.1(rng);
if let Some(result) = result {
return result;
} else {
choices[choice_index].0 -= 1;
}
}
}
/// pick is a helper function for uniformly picking a random element from a slice
pub(crate) fn pick<'a, T, R: Rng>(choices: &'a [T], rng: &mut R) -> &'a T {
let index = rng.gen_range(0..choices.len());

68
simulator/generation/plan.rs
View file

@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize};
use crate::{
model::{
query::{Create, Insert, Query, Select},
query::{Create, Delete, Distinctness, Insert, Query, Select},
table::Value,
},
runner::env::SimConnection,
@ -14,10 +14,7 @@ use crate::{
use crate::generation::{frequency, Arbitrary, ArbitraryFrom};
use super::{
pick,
property::{remaining, Property},
};
use super::property::{remaining, Property};
pub(crate) type ResultSet = Result<Vec<Vec<Value>>>;
@ -261,7 +258,7 @@ impl Interactions {
match self {
Interactions::Property(property) => {
match property {
Property::InsertSelect {
Property::InsertValuesSelect {
insert,
row_index: _,
queries,
@ -282,6 +279,32 @@ impl Interactions {
query.shadow(env);
}
}
Property::SelectLimit { select } => {
select.shadow(env);
}
Property::DeleteSelect {
table,
predicate,
queries,
} => {
let delete = Query::Delete(Delete {
table: table.clone(),
predicate: predicate.clone(),
});
let select = Query::Select(Select {
table: table.clone(),
predicate: predicate.clone(),
distinct: Distinctness::All,
limit: None,
});
delete.shadow(env);
for query in queries {
query.shadow(env);
}
select.shadow(env);
}
}
for interaction in property.interactions() {
match interaction {
@ -292,14 +315,26 @@ impl Interactions {
}
}
Query::Insert(insert) => {
let values = match &insert {
Insert::Values { values, .. } => values.clone(),
Insert::Select { select, .. } => select.shadow(env),
};
let table = env
.tables
.iter_mut()
.find(|t| t.name == insert.table)
.find(|t| t.name == insert.table())
.unwrap();
table.rows.extend(insert.values.clone());
table.rows.extend(values);
}
Query::Delete(delete) => {
let table = env
.tables
.iter_mut()
.find(|t| t.name == delete.table)
.unwrap();
let t2 = &table.clone();
table.rows.retain_mut(|r| delete.predicate.test(r, t2));
}
Query::Delete(_) => todo!(),
Query::Select(_) => {}
},
Interaction::Assertion(_) => {}
@ -308,7 +343,9 @@ impl Interactions {
}
}
}
Interactions::Query(query) => query.shadow(env),
Interactions::Query(query) => {
query.shadow(env);
}
Interactions::Fault(_) => {}
}
}
@ -389,12 +426,10 @@ impl ArbitraryFrom<&mut SimulatorEnv> for InteractionPlan {
}
impl Interaction {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) -> Vec<Vec<Value>> {
match self {
Self::Query(query) => query.shadow(env),
Self::Assumption(_) => {}
Self::Assertion(_) => {}
Self::Fault(_) => {}
Self::Assumption(_) | Self::Assertion(_) | Self::Fault(_) => vec![],
}
}
pub(crate) fn execute_query(&self, conn: &mut Rc<Connection>) -> ResultSet {
@ -547,12 +582,11 @@ fn create_table<R: rand::Rng>(rng: &mut R, _env: &SimulatorEnv) -> Interactions
}
fn random_read<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
Interactions::Query(Query::Select(Select::arbitrary_from(rng, &env.tables)))
Interactions::Query(Query::Select(Select::arbitrary_from(rng, env)))
}
fn random_write<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Interactions {
let table = pick(&env.tables, rng);
let insert_query = Query::Insert(Insert::arbitrary_from(rng, table));
let insert_query = Query::Insert(Insert::arbitrary_from(rng, env));
Interactions::Query(insert_query)
}

221
simulator/generation/property.rs
View file

@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
use crate::{
model::{
query::{Create, Delete, Insert, Predicate, Query, Select},
query::{Create, Delete, Distinctness, Insert, Predicate, Query, Select},
table::Value,
},
runner::env::SimulatorEnv,
@ -34,7 +34,7 @@ pub(crate) enum Property {
/// - The inserted row will not be deleted.
/// - The inserted row will not be updated.
/// - The table `t` will not be renamed, dropped, or altered.
InsertSelect {
InsertValuesSelect {
/// The insert query
insert: Insert,
/// Selected row index
@ -62,13 +62,47 @@ pub(crate) enum Property {
/// Additional interactions in the middle of the property
queries: Vec<Query>,
},
/// Select Limit is a property in which the select query
/// has a limit clause that is respected by the query.
/// The execution of the property is as follows
/// SELECT * FROM <t> WHERE <predicate> LIMIT <n>
/// This property is a single-interaction property.
/// The interaction has the following constraints;
/// - The select query will respect the limit clause.
SelectLimit {
/// The select query
select: Select,
},
/// Delete-Select is a property in which the deleted row
/// must not be in the resulting rows of a select query that has a
/// where clause that matches the deleted row. In practice, `p1` of
/// the delete query will be used as the predicate for the select query,
/// hence the select should return NO ROWS.
/// The execution of the property is as follows
/// DELETE FROM <t> WHERE <predicate>
/// I_0
/// I_1
/// ...
/// I_n
/// SELECT * FROM <t> WHERE <predicate>
/// The interactions in the middle has the following constraints;
/// - There will be no errors in the middle interactions.
/// - A row that holds for the predicate will not be inserted.
/// - The table `t` will not be renamed, dropped, or altered.
DeleteSelect {
table: String,
predicate: Predicate,
queries: Vec<Query>,
},
}
impl Property {
pub(crate) fn name(&self) -> String {
match self {
Property::InsertSelect { .. } => "Insert-Select".to_string(),
Property::InsertValuesSelect { .. } => "Insert-Values-Select".to_string(),
Property::DoubleCreateFailure { .. } => "Double-Create-Failure".to_string(),
Property::SelectLimit { .. } => "Select-Limit".to_string(),
Property::DeleteSelect { .. } => "Delete-Select".to_string(),
}
}
/// interactions construct a list of interactions, which is an executable representation of the property.
@ -76,26 +110,33 @@ impl Property {
/// and `interaction` cannot be serialized directly.
pub(crate) fn interactions(&self) -> Vec<Interaction> {
match self {
Property::InsertSelect {
Property::InsertValuesSelect {
insert,
row_index,
queries,
select,
} => {
let (table, values) = if let Insert::Values { table, values } = insert {
(table, values)
} else {
unreachable!(
"insert query should be Insert::Values for Insert-Values-Select property"
)
};
// Check that the insert query has at least 1 value
assert!(
!insert.values.is_empty(),
!values.is_empty(),
"insert query should have at least 1 value"
);
// Pick a random row within the insert values
let row = insert.values[*row_index].clone();
let row = values[*row_index].clone();
// Assume that the table exists
let assumption = Interaction::Assumption(Assertion {
message: format!("table {} exists", insert.table),
message: format!("table {} exists", insert.table()),
func: Box::new({
let table_name = insert.table.clone();
let table_name = table.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
}
@ -106,7 +147,7 @@ impl Property {
message: format!(
"row [{:?}] not found in table {}",
row.iter().map(|v| v.to_string()).collect::<Vec<String>>(),
insert.table,
insert.table(),
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let rows = stack.last().unwrap();
@ -162,12 +203,97 @@ impl Property {
interactions.push(cq2);
interactions.push(assertion);
interactions
}
Property::SelectLimit { select } => {
let table_name = select.table.clone();
let assumption = Interaction::Assumption(Assertion {
message: format!("table {} exists", table_name),
func: Box::new({
let table_name = table_name.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
}
}),
});
let limit = select
.limit
.expect("Property::SelectLimit without a LIMIT clause");
let assertion = Interaction::Assertion(Assertion {
message: "select query should respect the limit clause".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(rows) => Ok(limit >= rows.len()),
Err(_) => Ok(true),
}
}),
});
vec![
assumption,
Interaction::Query(Query::Select(select.clone())),
assertion,
]
}
Property::DeleteSelect {
table,
predicate,
queries,
} => {
let assumption = Interaction::Assumption(Assertion {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
let assertion = Interaction::Assertion(Assertion {
message: format!(
"select '{}' should return no values for table '{}'",
predicate, table,
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.is_empty()),
Err(err) => Err(LimboError::InternalError(err.to_string())),
}
}),
});
let delete = Interaction::Query(Query::Delete(Delete {
table: table.clone(),
predicate: predicate.clone(),
}));
let select = Interaction::Query(Query::Select(Select {
table: table.clone(),
predicate: predicate.clone(),
limit: None,
distinct: Distinctness::All,
}));
let mut interactions = Vec::new();
interactions.push(assumption);
interactions.push(delete);
interactions.extend(queries.clone().into_iter().map(Interaction::Query));
interactions.push(select);
interactions.push(assertion);
interactions
}
}
}
}
#[derive(Debug)]
pub(crate) struct Remaining {
pub(crate) read: f64,
pub(crate) write: f64,
@ -192,7 +318,7 @@ pub(crate) fn remaining(env: &SimulatorEnv, stats: &InteractionStats) -> Remaini
}
}
fn property_insert_select<R: rand::Rng>(
fn property_insert_values_select<R: rand::Rng>(
rng: &mut R,
env: &SimulatorEnv,
remaining: &Remaining,
@ -209,7 +335,7 @@ fn property_insert_select<R: rand::Rng>(
let row = rows[row_index].clone();
// Insert the rows
let insert_query = Insert {
let insert_query = Insert::Values {
table: table.name.clone(),
values: rows,
};
@ -221,7 +347,7 @@ fn property_insert_select<R: rand::Rng>(
// - [ ] The inserted row will not be updated. (todo: add this constraint once UPDATE is implemented)
// - [ ] The table `t` will not be renamed, dropped, or altered. (todo: add this constraint once ALTER or DROP is implemented)
for _ in 0..rng.gen_range(0..3) {
let query = Query::arbitrary_from(rng, (table, remaining));
let query = Query::arbitrary_from(rng, (env, remaining));
match &query {
Query::Delete(Delete {
table: t,
@ -248,9 +374,11 @@ fn property_insert_select<R: rand::Rng>(
let select_query = Select {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, (table, &row)),
limit: None,
distinct: Distinctness::All,
};
Property::InsertSelect {
Property::InsertValuesSelect {
insert: insert_query,
row_index,
queries,
@ -258,6 +386,19 @@ fn property_insert_select<R: rand::Rng>(
}
}
fn property_select_limit<R: rand::Rng>(rng: &mut R, env: &SimulatorEnv) -> Property {
// Get a random table
let table = pick(&env.tables, rng);
// Select the table
let select = Select {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, table),
limit: Some(rng.gen_range(1..=5)),
distinct: Distinctness::All,
};
Property::SelectLimit { select }
}
fn property_double_create_failure<R: rand::Rng>(
rng: &mut R,
env: &SimulatorEnv,
@ -276,7 +417,7 @@ fn property_double_create_failure<R: rand::Rng>(
// - [x] There will be no errors in the middle interactions.(best effort)
// - [ ] Table `t` will not be renamed or dropped.(todo: add this constraint once ALTER or DROP is implemented)
for _ in 0..rng.gen_range(0..3) {
let query = Query::arbitrary_from(rng, (table, remaining));
let query = Query::arbitrary_from(rng, (env, remaining));
match &query {
Query::Create(Create { table: t }) => {
// There will be no errors in the middle interactions.
@ -296,6 +437,48 @@ fn property_double_create_failure<R: rand::Rng>(
}
}
fn property_delete_select<R: rand::Rng>(
rng: &mut R,
env: &SimulatorEnv,
remaining: &Remaining,
) -> Property {
// Get a random table
let table = pick(&env.tables, rng);
// Generate a random predicate
let predicate = Predicate::arbitrary_from(rng, table);
// Create random queries respecting the constraints
let mut queries = Vec::new();
// - [x] There will be no errors in the middle interactions. (this constraint is impossible to check, so this is just best effort)
// - [x] A row that holds for the predicate will not be inserted.
// - [ ] The table `t` will not be renamed, dropped, or altered. (todo: add this constraint once ALTER or DROP is implemented)
for _ in 0..rng.gen_range(0..3) {
let query = Query::arbitrary_from(rng, (env, remaining));
match &query {
Query::Insert(Insert::Values { table: t, values }) => {
// A row that holds for the predicate will not be inserted.
if t == &table.name && values.iter().any(|v| predicate.test(v, table)) {
continue;
}
}
Query::Create(Create { table: t }) => {
// There will be no errors in the middle interactions.
// - Creating the same table is an error
if t.name == table.name {
continue;
}
}
_ => (),
}
queries.push(query);
}
Property::DeleteSelect {
table: table.name.clone(),
predicate,
queries,
}
}
impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property {
fn arbitrary_from<R: rand::Rng>(
rng: &mut R,
@ -306,12 +489,20 @@ impl ArbitraryFrom<(&SimulatorEnv, &InteractionStats)> for Property {
vec![
(
f64::min(remaining_.read, remaining_.write),
Box::new(|rng: &mut R| property_insert_select(rng, env, &remaining_)),
Box::new(|rng: &mut R| property_insert_values_select(rng, env, &remaining_)),
),
(
remaining_.create / 2.0,
Box::new(|rng: &mut R| property_double_create_failure(rng, env, &remaining_)),
),
(
remaining_.read,
Box::new(|rng: &mut R| property_select_limit(rng, env)),
),
(
f64::min(remaining_.read, remaining_.write),
Box::new(|rng: &mut R| property_delete_select(rng, env, &remaining_)),
),
],
rng,
)

186
simulator/generation/query.rs
View file

@ -1,13 +1,15 @@
use crate::generation::table::{GTValue, LTValue};
use crate::generation::{one_of, Arbitrary, ArbitraryFrom};
use crate::model::query::{Create, Delete, Insert, Predicate, Query, Select};
use crate::model::query::{Create, Delete, Distinctness, Insert, Predicate, Query, Select};
use crate::model::table::{Table, Value};
use crate::SimulatorEnv;
use rand::seq::SliceRandom as _;
use rand::Rng;
use super::property::Remaining;
use super::{frequency, pick};
use super::table::LikeValue;
use super::{backtrack, frequency, pick, ArbitraryFromMaybe};
impl Arbitrary for Create {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
@ -17,79 +19,85 @@ impl Arbitrary for Create {
}
}
impl ArbitraryFrom<&Vec<Table>> for Select {
fn arbitrary_from<R: Rng>(rng: &mut R, tables: &Vec<Table>) -> Self {
let table = pick(tables, rng);
impl ArbitraryFrom<&SimulatorEnv> for Select {
fn arbitrary_from<R: Rng>(rng: &mut R, env: &SimulatorEnv) -> Self {
let table = pick(&env.tables, rng);
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, table),
limit: Some(rng.gen_range(0..=1000)),
distinct: Distinctness::All,
}
}
}
impl ArbitraryFrom<&Vec<&Table>> for Select {
fn arbitrary_from<R: Rng>(rng: &mut R, tables: &Vec<&Table>) -> Self {
let table = pick(tables, rng);
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, *table),
}
}
}
impl ArbitraryFrom<&Table> for Insert {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
let num_rows = rng.gen_range(1..10);
let values: Vec<Vec<Value>> = (0..num_rows)
.map(|_| {
table
.columns
.iter()
.map(|c| Value::arbitrary_from(rng, &c.column_type))
.collect()
impl ArbitraryFrom<&SimulatorEnv> for Insert {
fn arbitrary_from<R: Rng>(rng: &mut R, env: &SimulatorEnv) -> Self {
let gen_values = |rng: &mut R| {
let table = pick(&env.tables, rng);
let num_rows = rng.gen_range(1..10);
let values: Vec<Vec<Value>> = (0..num_rows)
.map(|_| {
table
.columns
.iter()
.map(|c| Value::arbitrary_from(rng, &c.column_type))
.collect()
})
.collect();
Some(Insert::Values {
table: table.name.clone(),
values,
})
.collect();
Self {
table: table.name.clone(),
values,
}
}
}
};
impl ArbitraryFrom<&Table> for Delete {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, table),
}
}
}
let _gen_select = |rng: &mut R| {
// Find a non-empty table
let table = env.tables.iter().find(|t| !t.rows.is_empty());
if table.is_none() {
return None;
}
impl ArbitraryFrom<&Table> for Query {
fn arbitrary_from<R: Rng>(rng: &mut R, table: &Table) -> Self {
frequency(
let select_table = table.unwrap();
let row = pick(&select_table.rows, rng);
let predicate = Predicate::arbitrary_from(rng, (select_table, row));
// Pick another table to insert into
let select = Select {
table: select_table.name.clone(),
predicate,
limit: None,
distinct: Distinctness::All,
};
let table = pick(&env.tables, rng);
Some(Insert::Select {
table: table.name.clone(),
select: Box::new(select),
})
};
backtrack(
vec![
(1, Box::new(|rng| Self::Create(Create::arbitrary(rng)))),
(
100,
Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![table]))),
),
(
100,
Box::new(|rng| Self::Insert(Insert::arbitrary_from(rng, table))),
),
(
0,
Box::new(|rng| Self::Delete(Delete::arbitrary_from(rng, table))),
),
(1, Box::new(|rng| gen_values(rng))),
// todo: test and enable this once `INSERT INTO <table> SELECT * FROM <table>` is supported
// (1, Box::new(|rng| gen_select(rng))),
],
rng,
)
}
}
impl ArbitraryFrom<(&Table, &Remaining)> for Query {
fn arbitrary_from<R: Rng>(rng: &mut R, (table, remaining): (&Table, &Remaining)) -> Self {
impl ArbitraryFrom<&SimulatorEnv> for Delete {
fn arbitrary_from<R: Rng>(rng: &mut R, env: &SimulatorEnv) -> Self {
let table = pick(&env.tables, rng);
Self {
table: table.name.clone(),
predicate: Predicate::arbitrary_from(rng, table),
}
}
}
impl ArbitraryFrom<(&SimulatorEnv, &Remaining)> for Query {
fn arbitrary_from<R: Rng>(rng: &mut R, (env, remaining): (&SimulatorEnv, &Remaining)) -> Self {
frequency(
vec![
(
@ -98,15 +106,15 @@ impl ArbitraryFrom<(&Table, &Remaining)> for Query {
),
(
remaining.read,
Box::new(|rng| Self::Select(Select::arbitrary_from(rng, &vec![table]))),
Box::new(|rng| Self::Select(Select::arbitrary_from(rng, env))),
),
(
remaining.write,
Box::new(|rng| Self::Insert(Insert::arbitrary_from(rng, table))),
Box::new(|rng| Self::Insert(Insert::arbitrary_from(rng, env))),
),
(
0.0,
Box::new(|rng| Self::Delete(Delete::arbitrary_from(rng, table))),
remaining.write,
Box::new(|rng| Self::Delete(Delete::arbitrary_from(rng, env))),
),
],
rng,
@ -280,24 +288,48 @@ fn produce_true_predicate<R: Rng>(rng: &mut R, (t, row): (&Table, &Vec<Value>))
let column_index = rng.gen_range(0..t.columns.len());
let column = &t.columns[column_index];
let value = &row[column_index];
one_of(
backtrack(
vec![
Box::new(|_| Predicate::Eq(column.name.clone(), value.clone())),
Box::new(|rng| {
let v = loop {
(
1,
Box::new(|_| Some(Predicate::Eq(column.name.clone(), value.clone()))),
),
(
1,
Box::new(|rng| {
let v = Value::arbitrary_from(rng, &column.column_type);
if &v != value {
break v;
if &v == value {
None
} else {
Some(Predicate::Neq(column.name.clone(), v))
}
};
Predicate::Neq(column.name.clone(), v)
}),
Box::new(|rng| {
Predicate::Gt(column.name.clone(), LTValue::arbitrary_from(rng, value).0)
}),
Box::new(|rng| {
Predicate::Lt(column.name.clone(), GTValue::arbitrary_from(rng, value).0)
}),
}),
),
(
1,
Box::new(|rng| {
Some(Predicate::Gt(
column.name.clone(),
LTValue::arbitrary_from(rng, value).0,
))
}),
),
(
1,
Box::new(|rng| {
Some(Predicate::Lt(
column.name.clone(),
GTValue::arbitrary_from(rng, value).0,
))
}),
),
(
1,
Box::new(|rng| {
LikeValue::arbitrary_from_maybe(rng, value)
.map(|like| Predicate::Like(column.name.clone(), like.0))
}),
),
],
rng,
)

33
simulator/generation/table.rs
View file

@ -3,6 +3,8 @@ use rand::Rng;
use crate::generation::{gen_random_text, pick, readable_name_custom, Arbitrary, ArbitraryFrom};
use crate::model::table::{Column, ColumnType, Name, Table, Value};
use super::ArbitraryFromMaybe;
impl Arbitrary for Name {
fn arbitrary<R: Rng>(rng: &mut R) -> Self {
let name = readable_name_custom("_", rng);
@ -194,3 +196,34 @@ impl ArbitraryFrom<&Value> for GTValue {
}
}
}
pub(crate) struct LikeValue(pub(crate) String);
impl ArbitraryFromMaybe<&Value> for LikeValue {
fn arbitrary_from_maybe<R: Rng>(rng: &mut R, value: &Value) -> Option<Self> {
match value {
Value::Text(t) => {
let mut t = t.chars().collect::<Vec<_>>();
// Remove a number of characters, either insert `_` for each character removed, or
// insert one `%` for the whole substring
let mut i = 0;
while i < t.len() {
if rng.gen_bool(0.1) {
t[i] = '_';
} else if rng.gen_bool(0.05) {
t[i] = '%';
// skip a list of characters
for _ in 0..rng.gen_range(0..=3.min(t.len() - i - 1)) {
t.remove(i + 1);
}
}
i += 1;
}
let index = rng.gen_range(0..t.len());
t.insert(index, '%');
Some(Self(t.into_iter().collect()))
}
_ => None,
}
}
}

182
simulator/model/query.rs
View file

@ -1,5 +1,6 @@
use std::fmt::Display;
use regex::{Regex, RegexBuilder};
use serde::{Deserialize, Serialize};
use crate::{
@ -9,12 +10,48 @@ use crate::{
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub(crate) enum Predicate {
And(Vec<Predicate>), // p1 AND p2 AND p3... AND pn
Or(Vec<Predicate>), // p1 OR p2 OR p3... OR pn
Eq(String, Value), // column = Value
Neq(String, Value), // column != Value
Gt(String, Value), // column > Value
Lt(String, Value), // column < Value
And(Vec<Predicate>), // p1 AND p2 AND p3... AND pn
Or(Vec<Predicate>), // p1 OR p2 OR p3... OR pn
Eq(String, Value), // column = Value
Neq(String, Value), // column != Value
Gt(String, Value), // column > Value
Lt(String, Value), // column < Value
Like(String, String), // column LIKE Value
}
/// This function is a duplication of the exec_like function in core/vdbe/mod.rs at commit 9b9d5f9b4c9920e066ef1237c80878f4c3968524
/// Any updates to the original function should be reflected here, otherwise the test will be incorrect.
fn construct_like_regex(pattern: &str) -> Regex {
let mut regex_pattern = String::with_capacity(pattern.len() * 2);
regex_pattern.push('^');
for c in pattern.chars() {
match c {
'\\' => regex_pattern.push_str("\\\\"),
'%' => regex_pattern.push_str(".*"),
'_' => regex_pattern.push('.'),
ch => {
if regex_syntax::is_meta_character(c) {
regex_pattern.push('\\');
}
regex_pattern.push(ch);
}
}
}
regex_pattern.push('$');
RegexBuilder::new(&regex_pattern)
.case_insensitive(true)
.dot_matches_new_line(true)
.build()
.unwrap()
}
fn exec_like(pattern: &str, text: &str) -> bool {
let re = construct_like_regex(pattern);
re.is_match(text)
}
impl Predicate {
@ -43,6 +80,9 @@ impl Predicate {
Predicate::Neq(column, value) => get_value(column) != Some(value),
Predicate::Gt(column, value) => get_value(column).map(|v| v > value).unwrap_or(false),
Predicate::Lt(column, value) => get_value(column).map(|v| v < value).unwrap_or(false),
Predicate::Like(column, value) => get_value(column)
.map(|v| exec_like(v.to_string().as_str(), value.as_str()))
.unwrap_or(false),
}
}
}
@ -83,6 +123,7 @@ impl Display for Predicate {
Self::Neq(name, value) => write!(f, "{} != {}", name, value),
Self::Gt(name, value) => write!(f, "{} > {}", name, value),
Self::Lt(name, value) => write!(f, "{} < {}", name, value),
Self::Like(name, value) => write!(f, "{} LIKE '{}'", name, value),
}
}
}
@ -101,7 +142,8 @@ impl Query {
match self {
Query::Create(_) => vec![],
Query::Select(Select { table, .. })
| Query::Insert(Insert { table, .. })
| Query::Insert(Insert::Select { table, .. })
| Query::Insert(Insert::Values { table, .. })
| Query::Delete(Delete { table, .. }) => vec![table.clone()],
}
}
@ -109,12 +151,13 @@ impl Query {
match self {
Query::Create(Create { table }) => vec![table.name.clone()],
Query::Select(Select { table, .. })
| Query::Insert(Insert { table, .. })
| Query::Insert(Insert::Select { table, .. })
| Query::Insert(Insert::Values { table, .. })
| Query::Delete(Delete { table, .. }) => vec![table.clone()],
}
}
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) -> Vec<Vec<Value>> {
match self {
Query::Create(create) => create.shadow(env),
Query::Insert(insert) => insert.shadow(env),
@ -129,33 +172,110 @@ pub(crate) struct Create {
}
impl Create {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) -> Vec<Vec<Value>> {
if !env.tables.iter().any(|t| t.name == self.table.name) {
env.tables.push(self.table.clone());
}
vec![]
}
}
impl Display for Create {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "CREATE TABLE {} (", self.table.name)?;
for (i, column) in self.table.columns.iter().enumerate() {
if i != 0 {
write!(f, ",")?;
}
write!(f, "{} {}", column.name, column.column_type)?;
}
write!(f, ")")
}
}
/// `SELECT` distinctness
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum Distinctness {
/// `DISTINCT`
Distinct,
/// `ALL`
All,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub(crate) struct Select {
pub(crate) table: String,
pub(crate) predicate: Predicate,
pub(crate) distinct: Distinctness,
pub(crate) limit: Option<usize>,
}
impl Select {
pub(crate) fn shadow(&self, _env: &mut SimulatorEnv) {}
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) -> Vec<Vec<Value>> {
let table = env.tables.iter().find(|t| t.name == self.table.as_str());
if let Some(table) = table {
table
.rows
.iter()
.filter(|row| self.predicate.test(row, table))
.cloned()
.collect()
} else {
vec![]
}
}
}
impl Display for Select {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"SELECT * FROM {} WHERE {}{}",
self.table,
self.predicate,
self.limit
.map_or("".to_string(), |l| format!(" LIMIT {}", l))
)
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub(crate) struct Insert {
pub(crate) table: String,
pub(crate) values: Vec<Vec<Value>>,
pub(crate) enum Insert {
Values {
table: String,
values: Vec<Vec<Value>>,
},
Select {
table: String,
select: Box<Select>,
},
}
impl Insert {
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) {
if let Some(t) = env.tables.iter_mut().find(|t| t.name == self.table) {
t.rows.extend(self.values.clone());
pub(crate) fn shadow(&self, env: &mut SimulatorEnv) -> Vec<Vec<Value>> {
match self {
Insert::Values { table, values } => {
if let Some(t) = env.tables.iter_mut().find(|t| &t.name == table) {
t.rows.extend(values.clone());
}
}
Insert::Select { table, select } => {
let rows = select.shadow(env);
if let Some(t) = env.tables.iter_mut().find(|t| &t.name == table) {
t.rows.extend(rows);
}
}
}
vec![]
}
pub(crate) fn table(&self) -> &str {
match self {
Insert::Values { table, .. } | Insert::Select { table, .. } => table,
}
}
}
@ -167,31 +287,17 @@ pub(crate) struct Delete {
}
impl Delete {
pub(crate) fn shadow(&self, _env: &mut SimulatorEnv) {
todo!()
pub(crate) fn shadow(&self, _env: &mut SimulatorEnv) -> Vec<Vec<Value>> {
vec![]
}
}
impl Display for Query {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Create(Create { table }) => {
write!(f, "CREATE TABLE {} (", table.name)?;
for (i, column) in table.columns.iter().enumerate() {
if i != 0 {
write!(f, ",")?;
}
write!(f, "{} {}", column.name, column.column_type)?;
}
write!(f, ")")
}
Self::Select(Select {
table,
predicate: guard,
}) => write!(f, "SELECT * FROM {} WHERE {}", table, guard),
Self::Insert(Insert { table, values }) => {
Self::Create(create) => write!(f, "{}", create),
Self::Select(select) => write!(f, "{}", select),
Self::Insert(Insert::Values { table, values }) => {
write!(f, "INSERT INTO {} VALUES ", table)?;
for (i, row) in values.iter().enumerate() {
if i != 0 {
@ -208,6 +314,10 @@ impl Display for Query {
}
Ok(())
}
Self::Insert(Insert::Select { table, select }) => {
write!(f, "INSERT INTO {} ", table)?;
write!(f, "{}", select)
}
Self::Delete(Delete {
table,
predicate: guard,

13
simulator/shrink/plan.rs
View file

@ -1,5 +1,8 @@
use crate::{
generation::plan::{InteractionPlan, Interactions},
generation::{
plan::{InteractionPlan, Interactions},
property::Property,
},
model::query::Query,
runner::execution::Execution,
};
@ -27,12 +30,12 @@ impl InteractionPlan {
for interaction in plan.plan.iter_mut() {
if let Interactions::Property(p) = interaction {
match p {
crate::generation::property::Property::InsertSelect { queries, .. }
| crate::generation::property::Property::DoubleCreateFailure {
queries, ..
} => {
Property::InsertValuesSelect { queries, .. }
| Property::DoubleCreateFailure { queries, .. }
| Property::DeleteSelect { queries, .. } => {
queries.clear();
}
Property::SelectLimit { .. } => {}
}
}
}

35
testing/json.test
View file

@ -878,3 +878,38 @@ do_execsql_test json_set_add_array_in_array_in_nested_object {
do_execsql_test json_set_add_array_in_array_in_nested_object_out_of_bounds {
SELECT json_set('{}', '$.object[123].another', 'value', '$.field', 'value');
} {{{"field":"value"}}}
# The json_quote() function transforms an SQL value into a JSON value.
# String values are quoted and interior quotes are escaped. NULL values
# are rendered as the unquoted string "null".
#
do_execsql_test json_quote_string_literal {
SELECT json_quote('abc"xyz');
} {{"abc\"xyz"}}
do_execsql_test json_quote_float {
SELECT json_quote(3.14159);
} {3.14159}
do_execsql_test json_quote_integer {
SELECT json_quote(12345);
} {12345}
do_execsql_test json_quote_null {
SELECT json_quote(null);
} {"null"}
do_execsql_test json_quote_null_caps {
SELECT json_quote(NULL);
} null
do_execsql_test json_quote_json_value {
SELECT json_quote(json('{a:1, b: "test"}'));
} {{{"a":1,"b":"test"}}}
# Escape character tests in sqlite source depend on json_valid and in some syntax that is not implemented
# yet in limbo.
# See https://github.com/sqlite/sqlite/blob/255548562b125e6c148bb27d49aaa01b2fe61dba/test/json102.test#L690
# So for now not all control characters escaped are tested
# do_execsql_test json102-1501 {
# WITH RECURSIVE c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<0x1f)
# SELECT sum(json_valid(json_quote('a'||char(x)||'z'))) FROM c ORDER BY x;
# } {31}

16
vendored/sqlite3-parser/src/lexer/sql/mod.rs
View file

@ -30,6 +30,7 @@ pub struct Parser<'input> {
scanner: Scanner<Tokenizer>,
/// lemon parser
parser: yyParser<'input>,
had_error: bool,
}
impl<'input> Parser<'input> {
@ -43,12 +44,14 @@ impl<'input> Parser<'input> {
input,
scanner,
parser,
had_error: false,
}
}
/// Parse new `input`
pub fn reset(&mut self, input: &'input [u8]) {
self.input = input;
self.scanner.reset();
self.had_error = false;
}
/// Current line position in input
pub fn line(&self) -> u64 {
@ -182,6 +185,10 @@ impl FallibleIterator for Parser<'_> {
fn next(&mut self) -> Result<Option<Cmd>, Error> {
//print!("line: {}, column: {}: ", self.scanner.line(), self.scanner.column());
// if we have already encountered an error, return None to signal that to fallible_iterator that we are done parsing
if self.had_error {
return Ok(None);
}
self.parser.ctx.reset();
let mut last_token_parsed = TK_EOF;
let mut eof = false;
@ -197,6 +204,7 @@ impl FallibleIterator for Parser<'_> {
if token_type == TK_ILLEGAL {
// break out of parsing loop and return error
self.parser.sqlite3ParserFinalize();
self.had_error = true;
return Err(Error::UnrecognizedToken(
Some((self.scanner.line(), self.scanner.column())),
Some(start.into()),
@ -242,12 +250,18 @@ impl FallibleIterator for Parser<'_> {
self.parser
.sqlite3Parser(TK_SEMI, sentinel(self.input.len()))
);
if self.parser.ctx.error().is_some() {
self.had_error = true;
}
}
try_with_position!(
self.scanner,
self.parser
.sqlite3Parser(TK_EOF, sentinel(self.input.len()))
);
if self.parser.ctx.error().is_some() {
self.had_error = true;
}
}
self.parser.sqlite3ParserFinalize();
if let Some(e) = self.parser.ctx.error() {
@ -256,6 +270,7 @@ impl FallibleIterator for Parser<'_> {
Some((self.scanner.line(), self.scanner.column())),
Some((self.offset() - 1).into()),
);
self.had_error = true;
return Err(err);
}
let cmd = self.parser.ctx.cmd();
@ -266,6 +281,7 @@ impl FallibleIterator for Parser<'_> {
Some((self.scanner.line(), self.scanner.column())),
Some((self.offset() - 1).into()),
);
self.had_error = true;
return Err(err);
}
}

15
vendored/sqlite3-parser/src/lexer/sql/test.rs
View file

@ -338,6 +338,21 @@ fn qualified_table_name_within_triggers() {
);
}
#[test]
fn select_from_error_stops_at_first_error() {
let mut parser = Parser::new(b"SELECT FROM foo;");
// First next() call should return the first syntax error
let err = parser.next().unwrap_err();
assert!(matches!(err, Error::ParserError(_, _, _)));
// Second next() call should return Ok(None) since parsing should have stopped
assert_eq!(parser.next().unwrap(), None);
// Third next() call should also return Ok(None)
assert_eq!(parser.next().unwrap(), None);
}
#[test]
fn indexed_by_clause_within_triggers() {
expect_parser_err_msg(