Add tracing-appender to log traces to file asyncronously

Cargo.lock

@@ -583,6 +583,15 @@ dependencies = [
  "itertools",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.8.6"
@@ -1678,6 +1687,7 @@ dependencies = [
  "shlex",
  "syntect",
  "tracing",
+ "tracing-appender",
  "tracing-subscriber",
 ]
 
@@ -3472,6 +3482,18 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-appender"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
+dependencies = [
+ "crossbeam-channel",
+ "thiserror 1.0.69",
+ "time",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "tracing-attributes"
 version = "0.1.28"

cli/Cargo.toml

@@ -39,6 +39,7 @@ rustyline = { version = "15.0.0", default-features = true, features = [
 shlex = "1.3.0"
 syntect = "5.2.0"
 tracing = "0.1.41"
+tracing-appender = "0.2.3"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 
 

cli/app.rs

@@ -6,6 +6,8 @@ use crate::{
 };
 use comfy_table::{Attribute, Cell, CellAlignment, Color, ContentArrangement, Row, Table};
 use limbo_core::{Database, LimboError, OwnedValue, Statement, StepResult};
+use tracing_appender::non_blocking::WorkerGuard;
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 use clap::Parser;
 use rustyline::{history::DefaultHistory, Editor};
@@ -49,6 +51,8 @@ pub struct Opts {
     pub vfs: Option<String>,
     #[clap(long, help = "Enable experimental MVCC feature")]
     pub experimental_mvcc: bool,
+    #[clap(short = 't', long, help = "specify output file for log traces")]
+    pub tracing_output: Option<String>,
 }
 
 const PROMPT: &str = "limbo> ";
@@ -130,6 +134,8 @@ impl<'a> Limbo<'a> {
             })
             .expect("Error setting Ctrl-C handler");
         }
+        let sql = opts.sql.clone();
+        let quiet = opts.quiet;
         let mut app = Self {
             prompt: PROMPT.to_string(),
             io,
@@ -137,21 +143,25 @@ impl<'a> Limbo<'a> {
             conn,
             interrupt_count,
             input_buff: String::new(),
-            opts: Settings::from(&opts),
+            opts: Settings::from(opts),
             rl,
         };
-
-        if opts.sql.is_some() {
-            app.handle_first_input(opts.sql.as_ref().unwrap());
-        }
-        if !opts.quiet {
-            app.write_fmt(format_args!("Limbo v{}", env!("CARGO_PKG_VERSION")))?;
-            app.writeln("Enter \".help\" for usage hints.")?;
-            app.display_in_memory()?;
-        }
+        app.first_run(sql, quiet)?;
         Ok(app)
     }
 
+    fn first_run(&mut self, sql: Option<String>, quiet: bool) -> io::Result<()> {
+        if let Some(sql) = sql {
+            self.handle_first_input(&sql);
+        }
+        if !quiet {
+            self.write_fmt(format_args!("Limbo v{}", env!("CARGO_PKG_VERSION")))?;
+            self.writeln("Enter \".help\" for usage hints.")?;
+            self.display_in_memory()?;
+        }
+        Ok(())
+    }
+
     fn handle_first_input(&mut self, cmd: &str) {
         if cmd.trim().starts_with('.') {
             self.handle_dot_command(&cmd[1..]);
@@ -695,6 +705,32 @@ impl<'a> Limbo<'a> {
         Ok(())
     }
 
+    pub fn init_tracing(&mut self) -> Result<WorkerGuard, std::io::Error> {
+        let (non_blocking, guard) = if let Some(file) = &self.opts.tracing_output {
+            tracing_appender::non_blocking(
+                std::fs::File::options()
+                    .append(true)
+                    .create(true)
+                    .open(file)?,
+            )
+        } else {
+            tracing_appender::non_blocking(std::io::stderr())
+        };
+        if let Err(e) = tracing_subscriber::registry()
+            .with(
+                tracing_subscriber::fmt::layer()
+                    .with_writer(non_blocking)
+                    .with_line_number(true)
+                    .with_thread_ids(true),
+            )
+            .with(EnvFilter::from_default_env())
+            .try_init()
+        {
+            println!("Unable to setup tracing appender: {:?}", e);
+        }
+        Ok(guard)
+    }
+
     fn display_schema(&mut self, table: Option<&str>) -> anyhow::Result<()> {
         let sql = match table {
         Some(table_name) => format!(

cli/input.rs

@@ -81,28 +81,30 @@ pub struct Settings {
     pub echo: bool,
     pub is_stdout: bool,
     pub io: Io,
+    pub tracing_output: Option<String>,
 }
 
-impl From<&Opts> for Settings {
-    fn from(opts: &Opts) -> Self {
+impl From<Opts> for Settings {
+    fn from(opts: Opts) -> Self {
         Self {
             null_value: String::new(),
             output_mode: opts.output_mode,
             echo: false,
             is_stdout: opts.output.is_empty(),
-            output_filename: opts.output.clone(),
+            output_filename: opts.output,
             db_file: opts
                 .database
                 .as_ref()
                 .map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()),
             io: match opts.vfs.as_ref().unwrap_or(&String::new()).as_str() {
-                "memory" => Io::Memory,
+                "memory" | ":memory:" => Io::Memory,
                 "syscall" => Io::Syscall,
                 #[cfg(all(target_os = "linux", feature = "io_uring"))]
                 "io_uring" => Io::IoUring,
                 "" => Io::default(),
                 vfs => Io::External(vfs.to_string()),
             },
+            tracing_output: opts.tracing_output,
         }
     }
 }

cli/main.rs

@@ -7,7 +7,6 @@ mod opcodes_dictionary;
 
 use rustyline::{error::ReadlineError, Config, Editor};
 use std::sync::atomic::Ordering;
-use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter};
 
 fn rustyline_config() -> Config {
     Config::builder()
@@ -17,15 +16,8 @@ fn rustyline_config() -> Config {
 
 fn main() -> anyhow::Result<()> {
     let mut rl = Editor::with_config(rustyline_config())?;
-    tracing_subscriber::registry()
-        .with(
-            tracing_subscriber::fmt::layer()
-                .with_line_number(true)
-                .with_thread_ids(true),
-        )
-        .with(EnvFilter::from_default_env())
-        .init();
     let mut app = app::Limbo::new(&mut rl)?;
+    let _guard = app.init_tracing()?;
     let home = dirs::home_dir().expect("Could not determine home directory");
     let history_file = home.join(".limbo_history");
     if history_file.exists() {

testing/cli_tests/test_limbo_cli.py

@@ -11,7 +11,7 @@ PIPE_BUF = 4096
 
 
 class ShellConfig:
-    def __init__(self, exe_name, flags: str = "-q"):
+    def __init__(self, exe_name, flags: str = "-q -t testing/trace.log"):
         self.sqlite_exec: str = exe_name
         self.sqlite_flags: List[str] = flags.split()
         self.cwd = os.getcwd()