mirrors/limbo
1
0
Fork 0
mirror of https://github.com/tursodatabase/limbo.git synced 2025-08-04 18:18:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

add non-zero exit code in case of failures, remove the interactive initalization option in bug base for now, fix bugs in differential mode, add detailed information regarding runs to the bug base

Browse source
This commit is contained in:
alpaylan 2025-04-11 11:23:03 -04:00
parent 0bee24e7ad
commit d4707fe391
11 changed files with 517 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

6
Cargo.lock generated
View file

@ -344,6 +344,7 @@ dependencies = [
"iana-time-zone",
"js-sys",
"num-traits",
"serde",
"wasm-bindgen",
"windows-link",
]
@ -1866,6 +1867,7 @@ name = "limbo_sim"
version = "0.0.19-pre.4"
dependencies = [
"anarchist-readable-name-generator-lib",
"chrono",
"clap",
"dirs 6.0.0",
"env_logger 0.10.2",
@ -2292,9 +2294,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.21.1"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "onig"

1
simulator/Cargo.toml
View file

@ -31,3 +31,4 @@ serde_json = { version = "1.0" }
notify = "8.0.0"
rusqlite = { version = "0.34", features = ["bundled"] }
dirs = "6.0.0"
chrono = { version = "0.4.40", features = ["serde"] }

23
simulator/generation/plan.rs
View file

@ -11,7 +11,7 @@ use crate::{
},
table::Value,
},
runner::env::{SimConnection, SimulatorEnvTrait},
runner::env::SimConnection,
SimulatorEnv,
};
@ -238,7 +238,7 @@ impl Display for Interaction {
}
}
type AssertionFunc = dyn Fn(&Vec<ResultSet>, &dyn SimulatorEnvTrait) -> Result<bool>;
type AssertionFunc = dyn Fn(&Vec<ResultSet>, &SimulatorEnv) -> Result<bool>;
enum AssertionAST {
Pick(),
@ -523,7 +523,7 @@ impl Interaction {
pub(crate) fn execute_assertion(
&self,
stack: &Vec<ResultSet>,
env: &impl SimulatorEnvTrait,
env: &SimulatorEnv,
) -> Result<()> {
match self {
Self::Query(_) => {
@ -554,7 +554,7 @@ impl Interaction {
pub(crate) fn execute_assumption(
&self,
stack: &Vec<ResultSet>,
env: &dyn SimulatorEnvTrait,
env: &SimulatorEnv,
) -> Result<()> {
match self {
Self::Query(_) => {
@ -596,15 +596,12 @@ impl Interaction {
Self::Fault(fault) => {
match fault {
Fault::Disconnect => {
match env.connections[conn_index] {
SimConnection::Connected(ref mut conn) => {
conn.close()?;
}
SimConnection::Disconnected => {
return Err(limbo_core::LimboError::InternalError(
"Tried to disconnect a disconnected connection".to_string(),
));
}
if env.connections[conn_index].is_connected() {
env.connections[conn_index].disconnect();
} else {
return Err(limbo_core::LimboError::InternalError(
"connection already disconnected".into(),
));
}
env.connections[conn_index] = SimConnection::Disconnected;
}

40
simulator/generation/property.rs
View file

@ -9,7 +9,7 @@ use crate::{
},
table::Value,
},
runner::env::{SimulatorEnv, SimulatorEnvTrait},
runner::env::SimulatorEnv,
};
use super::{
@ -170,8 +170,8 @@ impl Property {
message: format!("table {} exists", insert.table()),
func: Box::new({
let table_name = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table_name))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
}
}),
});
@ -182,7 +182,7 @@ impl Property {
row.iter().map(|v| v.to_string()).collect::<Vec<String>>(),
insert.table(),
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.iter().any(|r| r == &row)),
@ -206,8 +206,8 @@ impl Property {
let assumption = Interaction::Assumption(Assertion {
message: "Double-Create-Failure should not be called on an existing table"
.to_string(),
func: Box::new(move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(!env.tables().iter().any(|t| t.name == table_name))
func: Box::new(move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(!env.tables.iter().any(|t| t.name == table_name))
}),
});
@ -220,11 +220,11 @@ impl Property {
message:
"creating two tables with the name should result in a failure for the second query"
.to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(_) => Ok(false),
Err(e) => Ok(e.to_string().contains(&format!("Table {table_name} already exists"))),
Err(e) => Ok(e.to_string().to_lowercase().contains(&format!("table {table_name} already exists"))),
}
}),
});
@ -245,8 +245,8 @@ impl Property {
message: format!("table {} exists", table_name),
func: Box::new({
let table_name = table_name.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table_name))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table_name))
}
}),
});
@ -257,7 +257,7 @@ impl Property {
let assertion = Interaction::Assertion(Assertion {
message: "select query should respect the limit clause".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(rows) => Ok(limit >= rows.len()),
@ -281,8 +281,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
@ -292,7 +292,7 @@ impl Property {
"select '{}' should return no values for table '{}'",
predicate, table,
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let rows = stack.last().unwrap();
match rows {
Ok(rows) => Ok(rows.is_empty()),
@ -332,8 +332,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
@ -345,7 +345,7 @@ impl Property {
"select query should result in an error for table '{}'",
table
),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let last = stack.last().unwrap();
match last {
Ok(_) => Ok(false),
@ -377,8 +377,8 @@ impl Property {
message: format!("table {} exists", table),
func: Box::new({
let table = table.clone();
move |_: &Vec<ResultSet>, env: &dyn SimulatorEnvTrait| {
Ok(env.tables().iter().any(|t| t.name == table))
move |_: &Vec<ResultSet>, env: &SimulatorEnv| {
Ok(env.tables.iter().any(|t| t.name == table))
}
}),
});
@ -401,7 +401,7 @@ impl Property {
let assertion = Interaction::Assertion(Assertion {
message: "select queries should return the same amount of results".to_string(),
func: Box::new(move |stack: &Vec<ResultSet>, _: &dyn SimulatorEnvTrait| {
func: Box::new(move |stack: &Vec<ResultSet>, _: &SimulatorEnv| {
let select_star = stack.last().unwrap();
let select_predicate = stack.get(stack.len() - 2).unwrap();
match (select_predicate, select_star) {

134
simulator/main.rs
View file

@ -5,7 +5,7 @@ use generation::ArbitraryFrom;
use notify::event::{DataChange, ModifyKind};
use notify::{EventKind, RecursiveMode, Watcher};
use rand::prelude::*;
use runner::bugbase::{Bug, BugBase};
use runner::bugbase::{Bug, BugBase, LoadedBug};
use runner::cli::SimulatorCLI;
use runner::env::SimulatorEnv;
use runner::execution::{execute_plans, Execution, ExecutionHistory, ExecutionResult};
@ -28,6 +28,7 @@ struct Paths {
history: PathBuf,
doublecheck_db: PathBuf,
shrunk_db: PathBuf,
diff_db: PathBuf,
}
impl Paths {
@ -40,6 +41,7 @@ impl Paths {
history: PathBuf::from(output_dir).join("history.txt"),
doublecheck_db: PathBuf::from(output_dir).join("double.db"),
shrunk_db: PathBuf::from(output_dir).join("shrunk.db"),
diff_db: PathBuf::from(output_dir).join("diff.db"),
}
}
}
@ -52,7 +54,6 @@ fn main() -> Result<(), String> {
let mut bugbase = BugBase::load().map_err(|e| format!("{:?}", e))?;
banner();
// let paths = Paths::new(&output_dir, cli_opts.doublecheck);
let last_execution = Arc::new(Mutex::new(Execution::new(0, 0, 0)));
let (seed, env, plans) = setup_simulation(&mut bugbase, &cli_opts, |p| &p.plan, |p| &p.db);
@ -66,8 +67,18 @@ fn main() -> Result<(), String> {
if cli_opts.watch {
watch_mode(seed, &cli_opts, &paths, last_execution.clone()).unwrap();
} else if cli_opts.differential {
differential_testing(env, plans, last_execution.clone())
return Ok(());
}
let result = if cli_opts.differential {
differential_testing(
seed,
&mut bugbase,
&cli_opts,
&paths,
plans,
last_execution.clone(),
)
} else {
run_simulator(
seed,
@ -77,13 +88,14 @@ fn main() -> Result<(), String> {
env,
plans,
last_execution.clone(),
);
}
)
};
// Print the seed, the locations of the database and the plan file at the end again for easily accessing them.
println!("seed: {}", seed);
println!("path: {}", paths.base.display());
Ok(())
result
}
fn watch_mode(
@ -120,7 +132,7 @@ fn watch_mode(
i.shadow(&mut env);
});
});
let env = Arc::new(Mutex::new(env.clone()));
let env = Arc::new(Mutex::new(env.clone_without_connections()));
watch::run_simulation(env, &mut [plan], last_execution.clone())
}),
last_execution.clone(),
@ -133,7 +145,6 @@ fn watch_mode(
SandboxedResult::Panicked { error, .. }
| SandboxedResult::FoundBug { error, .. } => {
log::error!("simulation failed: '{}'", error);
println!("simulation failed: '{}'", error);
}
}
}
@ -153,7 +164,7 @@ fn run_simulator(
env: SimulatorEnv,
plans: Vec<InteractionPlan>,
last_execution: Arc<Mutex<Execution>>,
) {
) -> Result<(), String> {
std::panic::set_hook(Box::new(move |info| {
log::error!("panic occurred");
@ -181,15 +192,15 @@ fn run_simulator(
if cli_opts.doublecheck {
let env = SimulatorEnv::new(seed, cli_opts, &paths.doublecheck_db);
let env = Arc::new(Mutex::new(env));
doublecheck(env, paths, &plans, last_execution.clone(), result);
doublecheck(env, paths, &plans, last_execution.clone(), result)
} else {
// No doublecheck, run shrinking if panicking or found a bug.
match &result {
SandboxedResult::Correct => {
log::info!("simulation succeeded");
println!("simulation succeeded");
// remove the bugbase entry
bugbase.remove_bug(seed).unwrap();
bugbase.mark_successful_run(seed, cli_opts).unwrap();
Ok(())
}
SandboxedResult::Panicked {
error,
@ -217,8 +228,6 @@ fn run_simulator(
}
log::error!("simulation failed: '{}'", error);
println!("simulation failed: '{}'", error);
log::info!("Starting to shrink");
let shrunk_plans = plans
@ -260,12 +269,21 @@ fn run_simulator(
) => {
if e1 != e2 {
log::error!("shrinking failed, the error was not properly reproduced");
bugbase.add_bug(seed, plans[0].clone()).unwrap();
bugbase
.add_bug(seed, plans[0].clone(), Some(error.clone()), cli_opts)
.unwrap();
Err(format!("failed with error: '{}'", error))
} else {
log::info!("shrinking succeeded");
println!("shrinking succeeded");
log::info!(
"shrinking succeeded, reduced the plan from {} to {}",
plans[0].plan.len(),
shrunk_plans[0].plan.len()
);
// Save the shrunk database
bugbase.add_bug(seed, shrunk_plans[0].clone()).unwrap();
bugbase
.add_bug(seed, shrunk_plans[0].clone(), Some(e1.clone()), cli_opts)
.unwrap();
Err(format!("failed with error: '{}'", e1))
}
}
(_, SandboxedResult::Correct) => {
@ -273,7 +291,10 @@ fn run_simulator(
}
_ => {
log::error!("shrinking failed, the error was not properly reproduced");
bugbase.add_bug(seed, plans[0].clone()).unwrap();
bugbase
.add_bug(seed, plans[0].clone(), Some(error.clone()), cli_opts)
.unwrap();
Err(format!("failed with error: '{}'", error))
}
}
}
@ -287,7 +308,7 @@ fn doublecheck(
plans: &[InteractionPlan],
last_execution: Arc<Mutex<Execution>>,
result: SandboxedResult,
) {
) -> Result<(), String> {
// Run the simulation again
let result2 = SandboxedResult::from(
std::panic::catch_unwind(|| {
@ -299,29 +320,47 @@ fn doublecheck(
match (result, result2) {
(SandboxedResult::Correct, SandboxedResult::Panicked { .. }) => {
log::error!("doublecheck failed! first run succeeded, but second run panicked.");
Err("doublecheck failed! first run succeeded, but second run panicked.".to_string())
}
(SandboxedResult::FoundBug { .. }, SandboxedResult::Panicked { .. }) => {
log::error!(
"doublecheck failed! first run failed an assertion, but second run panicked."
);
Err(
"doublecheck failed! first run failed an assertion, but second run panicked."
.to_string(),
)
}
(SandboxedResult::Panicked { .. }, SandboxedResult::Correct) => {
log::error!("doublecheck failed! first run panicked, but second run succeeded.");
Err("doublecheck failed! first run panicked, but second run succeeded.".to_string())
}
(SandboxedResult::Panicked { .. }, SandboxedResult::FoundBug { .. }) => {
log::error!(
"doublecheck failed! first run panicked, but second run failed an assertion."
);
Err(
"doublecheck failed! first run panicked, but second run failed an assertion."
.to_string(),
)
}
(SandboxedResult::Correct, SandboxedResult::FoundBug { .. }) => {
log::error!(
"doublecheck failed! first run succeeded, but second run failed an assertion."
);
Err(
"doublecheck failed! first run succeeded, but second run failed an assertion."
.to_string(),
)
}
(SandboxedResult::FoundBug { .. }, SandboxedResult::Correct) => {
log::error!(
"doublecheck failed! first run failed an assertion, but second run succeeded."
);
Err(
"doublecheck failed! first run failed an assertion, but second run succeeded."
.to_string(),
)
}
(SandboxedResult::Correct, SandboxedResult::Correct)
| (SandboxedResult::FoundBug { .. }, SandboxedResult::FoundBug { .. })
@ -331,33 +370,62 @@ fn doublecheck(
let doublecheck_db_bytes = std::fs::read(&paths.doublecheck_db).unwrap();
if db_bytes != doublecheck_db_bytes {
log::error!("doublecheck failed! database files are different.");
log::error!("current: {}", paths.db.display());
log::error!("doublecheck: {}", paths.doublecheck_db.display());
Err(
"doublecheck failed! database files are different, check binary diffs for more details.".to_string()
)
} else {
log::info!("doublecheck succeeded! database files are the same.");
println!("doublecheck succeeded! database files are the same.");
Ok(())
}
}
}
}
fn differential_testing(
env: SimulatorEnv,
seed: u64,
bugbase: &mut BugBase,
cli_opts: &SimulatorCLI,
paths: &Paths,
plans: Vec<InteractionPlan>,
last_execution: Arc<Mutex<Execution>>,
) {
let env = Arc::new(Mutex::new(env));
) -> Result<(), String> {
let env = Arc::new(Mutex::new(SimulatorEnv::new(seed, cli_opts, &paths.db)));
let rusqlite_env = Arc::new(Mutex::new(SimulatorEnv::new(
seed,
cli_opts,
&paths.diff_db,
)));
let result = SandboxedResult::from(
std::panic::catch_unwind(|| {
let plan = plans[0].clone();
differential::run_simulation(env, &mut [plan], last_execution.clone())
differential::run_simulation(
env,
rusqlite_env,
&|| rusqlite::Connection::open(paths.diff_db.clone()).unwrap(),
&mut [plan],
last_execution.clone(),
)
}),
last_execution.clone(),
);
if let SandboxedResult::Correct = result {
log::info!("simulation succeeded");
println!("simulation succeeded");
} else {
log::error!("simulation failed");
println!("simulation failed");
match result {
SandboxedResult::Correct => {
log::info!("simulation succeeded, output of Limbo conforms to SQLite");
println!("simulation succeeded, output of Limbo conforms to SQLite");
Ok(())
}
SandboxedResult::Panicked { error, .. } | SandboxedResult::FoundBug { error, .. } => {
log::error!("simulation failed: '{}'", error);
bugbase
.add_bug(seed, plans[0].clone(), Some(error.clone()), cli_opts)
.unwrap();
Err(format!("simulation failed: '{}'", error))
}
}
}
@ -433,12 +501,14 @@ fn setup_simulation(
let env = SimulatorEnv::new(bug.seed(), cli_opts, db_path(&paths));
let plan = match bug {
Bug::Loaded { plan, .. } => plan.clone(),
Bug::Loaded(LoadedBug { plan, .. }) => plan.clone(),
Bug::Unloaded { seed } => {
let seed = *seed;
bugbase
.load_bug(seed)
.unwrap_or_else(|_| panic!("could not load bug '{}' in bug base", seed))
.plan
.clone()
}
};

222
simulator/runner/bugbase.rs
View file

@ -3,15 +3,44 @@ use std::{
io::{self, Write},
path::PathBuf,
process::Command,
time::SystemTime,
};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use crate::{InteractionPlan, Paths};
use super::cli::SimulatorCLI;
/// A bug is a run that has been identified as buggy.
#[derive(Clone)]
pub(crate) enum Bug {
Unloaded { seed: u64 },
Loaded { seed: u64, plan: InteractionPlan },
Loaded(LoadedBug),
}
#[derive(Clone)]
pub struct LoadedBug {
/// The seed of the bug.
pub seed: u64,
/// The plan of the bug.
pub plan: InteractionPlan,
/// The runs of the bug.
pub runs: Vec<BugRun>,
}
#[derive(Clone, Serialize, Deserialize)]
pub(crate) struct BugRun {
/// Commit hash of the current version of Limbo.
hash: String,
/// Timestamp of the run.
#[serde(with = "chrono::serde::ts_seconds")]
timestamp: DateTime<Utc>,
/// Error message of the run.
error: Option<String>,
/// Options
cli_options: SimulatorCLI,
}
impl Bug {
@ -27,7 +56,7 @@ impl Bug {
pub(crate) fn seed(&self) -> u64 {
match self {
Bug::Unloaded { seed } => *seed,
Bug::Loaded { seed, .. } => *seed,
Bug::Loaded(LoadedBug { seed, .. }) => *seed,
}
}
}
@ -77,6 +106,36 @@ impl BugBase {
.or(Err("should be able to get current directory".to_string()))?,
];
for path in &potential_paths {
let path = path.join(".bugbase");
if path.exists() {
return BugBase::new(path);
}
}
for path in potential_paths {
let path = path.join(".bugbase");
if std::fs::create_dir_all(&path).is_ok() {
log::info!("bug base created at {}", path.display());
return BugBase::new(path);
}
}
Err("failed to create bug base".to_string())
}
/// Load the bug base from one of the potential paths.
pub(crate) fn interactive_load() -> Result<Self, String> {
let potential_paths = vec![
// limbo project directory
BugBase::get_limbo_project_dir()?,
// home directory
dirs::home_dir().ok_or("should be able to get home directory".to_string())?,
// current directory
std::env::current_dir()
.or(Err("should be able to get current directory".to_string()))?,
];
for path in potential_paths {
let path = path.join(".bugbase");
if path.exists() {
@ -119,14 +178,41 @@ impl BugBase {
}
/// Add a new bug to the bug base.
pub(crate) fn add_bug(&mut self, seed: u64, plan: InteractionPlan) -> Result<(), String> {
pub(crate) fn add_bug(
&mut self,
seed: u64,
plan: InteractionPlan,
error: Option<String>,
cli_options: &SimulatorCLI,
) -> Result<(), String> {
log::debug!("adding bug with seed {}", seed);
if self.bugs.contains_key(&seed) {
return Err(format!("Bug with hash {} already exists", seed));
let bug = self.get_bug(seed);
if bug.is_some() {
let mut bug = self.load_bug(seed)?;
bug.plan = plan.clone();
bug.runs.push(BugRun {
hash: Self::get_current_commit_hash()?,
timestamp: SystemTime::now().into(),
error,
cli_options: cli_options.clone(),
});
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
} else {
let bug = LoadedBug {
seed,
plan: plan.clone(),
runs: vec![BugRun {
hash: Self::get_current_commit_hash()?,
timestamp: SystemTime::now().into(),
error,
cli_options: cli_options.clone(),
}],
};
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
}
self.save_bug(seed, &plan)?;
self.bugs.insert(seed, Bug::Loaded { seed, plan });
Ok(())
// Save the bug to the bug base.
self.save_bug(seed)
}
/// Get a bug from the bug base.
@ -135,36 +221,48 @@ impl BugBase {
}
/// Save a bug to the bug base.
pub(crate) fn save_bug(&self, seed: u64, plan: &InteractionPlan) -> Result<(), String> {
let bug_path = self.path.join(seed.to_string());
std::fs::create_dir_all(&bug_path)
.or(Err("should be able to create bug directory".to_string()))?;
fn save_bug(&self, seed: u64) -> Result<(), String> {
let bug = self.get_bug(seed);
let seed_path = bug_path.join("seed.txt");
std::fs::write(&seed_path, seed.to_string())
.or(Err("should be able to write seed file".to_string()))?;
match bug {
None | Some(Bug::Unloaded { .. }) => {
unreachable!("save should only be called within add_bug");
}
Some(Bug::Loaded(bug)) => {
let bug_path = self.path.join(seed.to_string());
std::fs::create_dir_all(&bug_path)
.or(Err("should be able to create bug directory".to_string()))?;
// At some point we might want to save the commit hash of the current
// version of Limbo.
// let commit_hash = Self::get_current_commit_hash()?;
// let commit_hash_path = bug_path.join("commit_hash.txt");
// std::fs::write(&commit_hash_path, commit_hash)
// .or(Err("should be able to write commit hash file".to_string()))?;
let seed_path = bug_path.join("seed.txt");
std::fs::write(&seed_path, seed.to_string())
.or(Err("should be able to write seed file".to_string()))?;
let plan_path = bug_path.join("plan.json");
std::fs::write(
&plan_path,
serde_json::to_string(plan).or(Err("should be able to serialize plan".to_string()))?,
)
.or(Err("should be able to write plan file".to_string()))?;
let plan_path = bug_path.join("plan.json");
std::fs::write(
&plan_path,
serde_json::to_string_pretty(&bug.plan)
.or(Err("should be able to serialize plan".to_string()))?,
)
.or(Err("should be able to write plan file".to_string()))?;
let readable_plan_path = bug_path.join("plan.sql");
std::fs::write(&readable_plan_path, bug.plan.to_string())
.or(Err("should be able to write readable plan file".to_string()))?;
let runs_path = bug_path.join("runs.json");
std::fs::write(
&runs_path,
serde_json::to_string_pretty(&bug.runs)
.or(Err("should be able to serialize runs".to_string()))?,
)
.or(Err("should be able to write runs file".to_string()))?;
}
}
let readable_plan_path = bug_path.join("plan.sql");
std::fs::write(&readable_plan_path, plan.to_string())
.or(Err("should be able to write readable plan file".to_string()))?;
Ok(())
}
pub(crate) fn load_bug(&mut self, seed: u64) -> Result<InteractionPlan, String> {
pub(crate) fn load_bug(&mut self, seed: u64) -> Result<LoadedBug, String> {
let seed_match = self.bugs.get(&seed);
match seed_match {
@ -176,30 +274,60 @@ impl BugBase {
let plan: InteractionPlan = serde_json::from_str(&plan)
.or(Err("should be able to deserialize plan".to_string()))?;
let bug = Bug::Loaded {
let runs =
std::fs::read_to_string(self.path.join(seed.to_string()).join("runs.json"))
.or(Err("should be able to read runs file".to_string()))?;
let runs: Vec<BugRun> = serde_json::from_str(&runs)
.or(Err("should be able to deserialize runs".to_string()))?;
let bug = LoadedBug {
seed,
plan: plan.clone(),
runs,
};
self.bugs.insert(seed, bug);
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
log::debug!("Loaded bug with seed {}", seed);
Ok(plan)
Ok(bug)
}
Some(Bug::Loaded { plan, .. }) => {
Some(Bug::Loaded(bug)) => {
log::warn!(
"Bug with seed {} is already loaded, returning the existing plan",
seed
);
Ok(plan.clone())
Ok(bug.clone())
}
}
}
pub(crate) fn remove_bug(&mut self, seed: u64) -> Result<(), String> {
self.bugs.remove(&seed);
std::fs::remove_dir_all(self.path.join(seed.to_string()))
.or(Err("should be able to remove bug directory".to_string()))?;
pub(crate) fn mark_successful_run(
&mut self,
seed: u64,
cli_options: &SimulatorCLI,
) -> Result<(), String> {
let bug = self.get_bug(seed);
match bug {
None => {
log::debug!("removing bug base entry for {}", seed);
std::fs::remove_dir_all(self.path.join(seed.to_string()))
.or(Err("should be able to remove bug directory".to_string()))?;
}
Some(_) => {
let mut bug = self.load_bug(seed)?;
bug.runs.push(BugRun {
hash: Self::get_current_commit_hash()?,
timestamp: SystemTime::now().into(),
error: None,
cli_options: cli_options.clone(),
});
self.bugs.insert(seed, Bug::Loaded(bug.clone()));
// Save the bug to the bug base.
self.save_bug(seed)
.or(Err("should be able to save bug".to_string()))?;
log::debug!("Updated bug with seed {}", seed);
}
}
log::debug!("Removed bug with seed {}", seed);
Ok(())
}
}
@ -223,6 +351,18 @@ impl BugBase {
}
impl BugBase {
pub(crate) fn get_current_commit_hash() -> Result<String, String> {
let output = Command::new("git")
.args(["rev-parse", "HEAD"])
.output()
.or(Err("should be able to get the commit hash".to_string()))?;
let commit_hash = String::from_utf8(output.stdout)
.or(Err("commit hash should be valid utf8".to_string()))?
.trim()
.to_string();
Ok(commit_hash)
}
pub(crate) fn get_limbo_project_dir() -> Result<PathBuf, String> {
Ok(PathBuf::from(
String::from_utf8(

3
simulator/runner/cli.rs
View file

@ -1,6 +1,7 @@
use clap::{command, Parser};
use serde::{Deserialize, Serialize};
#[derive(Parser)]
#[derive(Parser, Clone, Serialize, Deserialize)]
#[command(name = "limbo-simulator")]
#[command(author, version, about, long_about = None)]
pub struct SimulatorCLI {

238
simulator/runner/differential.rs
View file

@ -5,54 +5,20 @@ use crate::{
pick_index,
plan::{Interaction, InteractionPlanState, ResultSet},
},
model::{
query::Query,
table::{Table, Value},
},
model::{query::Query, table::Value},
runner::execution::ExecutionContinuation,
InteractionPlan,
};
use super::{
env::{ConnectionTrait, SimConnection, SimulatorEnv, SimulatorEnvTrait},
env::{SimConnection, SimulatorEnv},
execution::{execute_interaction, Execution, ExecutionHistory, ExecutionResult},
};
pub(crate) struct SimulatorEnvRusqlite {
pub(crate) tables: Vec<Table>,
pub(crate) connections: Vec<RusqliteConnection>,
}
pub(crate) enum RusqliteConnection {
Connected(rusqlite::Connection),
Disconnected,
}
impl ConnectionTrait for RusqliteConnection {
fn is_connected(&self) -> bool {
match self {
RusqliteConnection::Connected(_) => true,
RusqliteConnection::Disconnected => false,
}
}
fn disconnect(&mut self) {
*self = RusqliteConnection::Disconnected;
}
}
impl SimulatorEnvTrait for SimulatorEnvRusqlite {
fn tables(&self) -> &Vec<Table> {
&self.tables
}
fn tables_mut(&mut self) -> &mut Vec<Table> {
&mut self.tables
}
}
pub(crate) fn run_simulation(
env: Arc<Mutex<SimulatorEnv>>,
rusqlite_env: Arc<Mutex<SimulatorEnv>>,
rusqlite_conn: &dyn Fn() -> rusqlite::Connection,
plans: &mut [InteractionPlan],
last_execution: Arc<Mutex<Execution>>,
) -> ExecutionResult {
@ -66,14 +32,7 @@ pub(crate) fn run_simulation(
secondary_pointer: 0,
})
.collect::<Vec<_>>();
let env = env.lock().unwrap();
let rusqlite_env = SimulatorEnvRusqlite {
tables: env.tables.clone(),
connections: (0..env.connections.len())
.map(|_| RusqliteConnection::Connected(rusqlite::Connection::open_in_memory().unwrap()))
.collect::<Vec<_>>(),
};
let mut rusqlite_states = plans
.iter()
.map(|_| InteractionPlanState {
@ -84,16 +43,15 @@ pub(crate) fn run_simulation(
.collect::<Vec<_>>();
let result = execute_plans(
Arc::new(Mutex::new(env.clone())),
env,
rusqlite_env,
rusqlite_conn,
plans,
&mut states,
&mut rusqlite_states,
last_execution,
);
env.io.print_stats();
log::info!("Simulation completed");
result
@ -148,7 +106,8 @@ fn execute_query_rusqlite(
pub(crate) fn execute_plans(
env: Arc<Mutex<SimulatorEnv>>,
mut rusqlite_env: SimulatorEnvRusqlite,
rusqlite_env: Arc<Mutex<SimulatorEnv>>,
rusqlite_conn: &dyn Fn() -> rusqlite::Connection,
plans: &mut [InteractionPlan],
states: &mut [InteractionPlanState],
rusqlite_states: &mut [InteractionPlanState],
@ -158,6 +117,8 @@ pub(crate) fn execute_plans(
let now = std::time::Instant::now();
let mut env = env.lock().unwrap();
let mut rusqlite_env = rusqlite_env.lock().unwrap();
for _tick in 0..env.opts.ticks {
// Pick the connection to interact with
let connection_index = pick_index(env.connections.len(), &mut env.rng);
@ -176,6 +137,7 @@ pub(crate) fn execute_plans(
match execute_plan(
&mut env,
&mut rusqlite_env,
rusqlite_conn,
connection_index,
plans,
states,
@ -202,13 +164,15 @@ pub(crate) fn execute_plans(
fn execute_plan(
env: &mut SimulatorEnv,
rusqlite_env: &mut SimulatorEnvRusqlite,
rusqlite_env: &mut SimulatorEnv,
rusqlite_conn: &dyn Fn() -> rusqlite::Connection,
connection_index: usize,
plans: &mut [InteractionPlan],
states: &mut [InteractionPlanState],
rusqlite_states: &mut [InteractionPlanState],
) -> limbo_core::Result<()> {
let connection = &env.connections[connection_index];
let rusqlite_connection = &rusqlite_env.connections[connection_index];
let plan = &mut plans[connection_index];
let state = &mut states[connection_index];
let rusqlite_state = &mut rusqlite_states[connection_index];
@ -218,83 +182,141 @@ fn execute_plan(
let interaction = &plan.plan[state.interaction_pointer].interactions()[state.secondary_pointer];
if let SimConnection::Disconnected = connection {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect().unwrap());
} else {
let limbo_result =
execute_interaction(env, connection_index, interaction, &mut state.stack);
let ruqlite_result = execute_interaction_rusqlite(
rusqlite_env,
connection_index,
interaction,
&mut rusqlite_state.stack,
);
match (connection, rusqlite_connection) {
(SimConnection::Disconnected, SimConnection::Disconnected) => {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] =
SimConnection::LimboConnection(env.db.connect().unwrap());
rusqlite_env.connections[connection_index] =
SimConnection::SQLiteConnection(rusqlite_conn());
}
(SimConnection::LimboConnection(_), SimConnection::SQLiteConnection(_)) => {
let limbo_result =
execute_interaction(env, connection_index, interaction, &mut state.stack);
let ruqlite_result = execute_interaction_rusqlite(
rusqlite_env,
connection_index,
interaction,
&mut rusqlite_state.stack,
);
match (limbo_result, ruqlite_result) {
(Ok(next_execution), Ok(next_execution_rusqlite)) => {
if next_execution != next_execution_rusqlite {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
match (limbo_result, ruqlite_result) {
(Ok(next_execution), Ok(next_execution_rusqlite)) => {
if next_execution != next_execution_rusqlite {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
log::debug!("connection {} processed", connection_index);
// Move to the next interaction or property
match next_execution {
ExecutionContinuation::NextInteraction => {
if state.secondary_pointer + 1
>= plan.plan[state.interaction_pointer].interactions().len()
{
// If we have reached the end of the interactions for this property, move to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
} else {
// Otherwise, move to the next interaction
state.secondary_pointer += 1;
let limbo_values = state.stack.last();
let rusqlite_values = rusqlite_state.stack.last();
match (limbo_values, rusqlite_values) {
(Some(limbo_values), Some(rusqlite_values)) => {
match (limbo_values, rusqlite_values) {
(Ok(limbo_values), Ok(rusqlite_values)) => {
if limbo_values != rusqlite_values {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
}
(Err(limbo_err), Err(rusqlite_err)) => {
log::warn!(
"limbo and rusqlite both fail, requires manual check"
);
log::warn!("limbo error {}", limbo_err);
log::warn!("rusqlite error {}", rusqlite_err);
}
(Ok(limbo_result), Err(rusqlite_err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo values {:?}", limbo_result);
log::error!("rusqlite error {}", rusqlite_err);
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
(Err(limbo_err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", limbo_err);
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
}
}
(None, None) => {}
_ => {
log::error!("limbo and rusqlite results do not match");
return Err(limbo_core::LimboError::InternalError(
"limbo and rusqlite results do not match".into(),
));
}
}
ExecutionContinuation::NextProperty => {
// Skip to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
// Move to the next interaction or property
match next_execution {
ExecutionContinuation::NextInteraction => {
if state.secondary_pointer + 1
>= plan.plan[state.interaction_pointer].interactions().len()
{
// If we have reached the end of the interactions for this property, move to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
} else {
// Otherwise, move to the next interaction
state.secondary_pointer += 1;
}
}
ExecutionContinuation::NextProperty => {
// Skip to the next property
state.interaction_pointer += 1;
state.secondary_pointer = 0;
}
}
}
}
(Err(err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", err);
return Err(err);
}
(Ok(_), Err(err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("rusqlite error {}", err);
return Err(err);
}
(Err(err), Err(err_rusqlite)) => {
log::error!("limbo and rusqlite both fail, requires manual check");
log::error!("limbo error {}", err);
log::error!("rusqlite error {}", err_rusqlite);
return Err(err);
(Err(err), Ok(_)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo error {}", err);
return Err(err);
}
(Ok(val), Err(err)) => {
log::error!("limbo and rusqlite results do not match");
log::error!("limbo {:?}", val);
log::error!("rusqlite error {}", err);
return Err(err);
}
(Err(err), Err(err_rusqlite)) => {
log::error!("limbo and rusqlite both fail, requires manual check");
log::error!("limbo error {}", err);
log::error!("rusqlite error {}", err_rusqlite);
return Err(err);
}
}
}
_ => unreachable!("{} vs {}", connection, rusqlite_connection),
}
Ok(())
}
fn execute_interaction_rusqlite(
env: &mut SimulatorEnvRusqlite,
env: &mut SimulatorEnv,
connection_index: usize,
interaction: &Interaction,
stack: &mut Vec<ResultSet>,
) -> limbo_core::Result<ExecutionContinuation> {
log::info!("executing in rusqlite: {}", interaction);
log::trace!(
"execute_interaction_rusqlite(connection_index={}, interaction={})",
connection_index,
interaction
);
match interaction {
Interaction::Query(query) => {
let conn = match &mut env.connections[connection_index] {
RusqliteConnection::Connected(conn) => conn,
RusqliteConnection::Disconnected => unreachable!(),
SimConnection::SQLiteConnection(conn) => conn,
SimConnection::LimboConnection(_) => unreachable!(),
SimConnection::Disconnected => unreachable!(),
};
log::debug!("{}", interaction);
@ -318,7 +340,7 @@ fn execute_interaction_rusqlite(
}
}
Interaction::Fault(_) => {
log::debug!("faults are not supported in differential testing mode");
interaction.execute_fault(env, connection_index)?;
}
}

83
simulator/runner/env.rs
View file

@ -1,8 +1,10 @@
use std::fmt::Display;
use std::mem;
use std::path::Path;
use std::rc::Rc;
use std::sync::Arc;
use limbo_core::{Connection, Database};
use limbo_core::Database;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha8Rng;
@ -12,12 +14,6 @@ use crate::runner::io::SimulatorIO;
use super::cli::SimulatorCLI;
pub trait SimulatorEnvTrait {
fn tables(&self) -> &Vec<Table>;
fn tables_mut(&mut self) -> &mut Vec<Table>;
}
#[derive(Clone)]
pub(crate) struct SimulatorEnv {
pub(crate) opts: SimulatorOpts,
pub(crate) tables: Vec<Table>,
@ -27,13 +23,18 @@ pub(crate) struct SimulatorEnv {
pub(crate) rng: ChaCha8Rng,
}
impl SimulatorEnvTrait for SimulatorEnv {
fn tables(&self) -> &Vec<Table> {
&self.tables
}
fn tables_mut(&mut self) -> &mut Vec<Table> {
&mut self.tables
impl SimulatorEnv {
pub(crate) fn clone_without_connections(&self) -> Self {
SimulatorEnv {
opts: self.opts.clone(),
tables: self.tables.clone(),
connections: (0..self.connections.len())
.map(|_| SimConnection::Disconnected)
.collect(),
io: self.io.clone(),
db: self.db.clone(),
rng: self.rng.clone(),
}
}
}
@ -85,7 +86,11 @@ impl SimulatorEnv {
// Remove existing database file if it exists
if db_path.exists() {
std::fs::remove_file(db_path).unwrap();
std::fs::remove_file(db_path.with_extension("db-wal")).unwrap();
}
let wal_path = db_path.with_extension("db-wal");
if wal_path.exists() {
std::fs::remove_file(wal_path).unwrap();
}
let db = match Database::open_file(io.clone(), db_path.to_str().unwrap(), false) {
@ -95,7 +100,9 @@ impl SimulatorEnv {
}
};
let connections = vec![SimConnection::Disconnected; opts.max_connections];
let connections = (0..opts.max_connections)
.map(|_| SimConnection::Disconnected)
.collect::<Vec<_>>();
SimulatorEnv {
opts,
@ -108,27 +115,55 @@ impl SimulatorEnv {
}
}
pub trait ConnectionTrait {
pub trait ConnectionTrait
where
Self: std::marker::Sized + Clone,
{
fn is_connected(&self) -> bool;
fn disconnect(&mut self);
}
#[derive(Clone)]
pub(crate) enum SimConnection {
Connected(Rc<Connection>),
LimboConnection(Rc<limbo_core::Connection>),
SQLiteConnection(rusqlite::Connection),
Disconnected,
}
impl ConnectionTrait for SimConnection {
fn is_connected(&self) -> bool {
impl SimConnection {
pub(crate) fn is_connected(&self) -> bool {
match self {
SimConnection::Connected(_) => true,
SimConnection::LimboConnection(_) | SimConnection::SQLiteConnection(_) => true,
SimConnection::Disconnected => false,
}
}
pub(crate) fn disconnect(&mut self) {
let conn = mem::replace(self, SimConnection::Disconnected);
fn disconnect(&mut self) {
*self = SimConnection::Disconnected;
match conn {
SimConnection::LimboConnection(conn) => {
conn.close().unwrap();
}
SimConnection::SQLiteConnection(conn) => {
conn.close().unwrap();
}
SimConnection::Disconnected => {}
}
}
}
impl Display for SimConnection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SimConnection::LimboConnection(_) => {
write!(f, "LimboConnection")
}
SimConnection::SQLiteConnection(_) => {
write!(f, "SQLiteConnection")
}
SimConnection::Disconnected => {
write!(f, "Disconnected")
}
}
}
}

9
simulator/runner/execution.rs
View file

@ -122,8 +122,10 @@ fn execute_plan(
if let SimConnection::Disconnected = connection {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect().unwrap());
env.connections[connection_index] =
SimConnection::LimboConnection(env.db.connect().unwrap());
} else {
log::debug!("connection {} already connected", connection_index);
match execute_interaction(env, connection_index, interaction, &mut state.stack) {
Ok(next_execution) => {
interaction.shadow(env);
@ -163,7 +165,7 @@ fn execute_plan(
/// `execute_interaction` uses this type in conjunction with a result, where
/// the `Err` case indicates a full-stop due to a bug, and the `Ok` case
/// indicates the next step in the plan.
#[derive(PartialEq)]
#[derive(PartialEq, Debug)]
pub(crate) enum ExecutionContinuation {
/// Default continuation, execute the next interaction.
NextInteraction,
@ -185,7 +187,8 @@ pub(crate) fn execute_interaction(
match interaction {
Interaction::Query(_) => {
let conn = match &mut env.connections[connection_index] {
SimConnection::Connected(conn) => conn,
SimConnection::LimboConnection(conn) => conn,
SimConnection::SQLiteConnection(_) => unreachable!(),
SimConnection::Disconnected => unreachable!(),
};

3
simulator/runner/watch.rs
View file

@ -98,7 +98,8 @@ fn execute_plan(
if let SimConnection::Disconnected = connection {
log::debug!("connecting {}", connection_index);
env.connections[connection_index] = SimConnection::Connected(env.db.connect().unwrap());
env.connections[connection_index] =
SimConnection::LimboConnection(env.db.connect().unwrap());
} else {
match execute_interaction(env, connection_index, interaction, &mut state.stack) {
Ok(next_execution) => {