From 75385e025434879debcf699bd0728853c14caad1 Mon Sep 17 00:00:00 2001 From: Josh Date: Sun, 24 Aug 2025 19:43:51 -0500 Subject: [PATCH 01/30] workspace --- Cargo.lock | 279 ++++++++++++++++++++++++++++ Cargo.toml | 4 + crates/djls-server/Cargo.toml | 4 + crates/djls-workspace/Cargo.toml | 15 ++ crates/djls-workspace/src/bridge.rs | 108 +++++++++++ crates/djls-workspace/src/db.rs | 91 +++++++++ crates/djls-workspace/src/lib.rs | 30 +++ crates/djls-workspace/src/vfs.rs | 245 ++++++++++++++++++++++++ 8 files changed, 776 insertions(+) create mode 100644 crates/djls-workspace/Cargo.toml create mode 100644 crates/djls-workspace/src/bridge.rs create mode 100644 crates/djls-workspace/src/db.rs create mode 100644 crates/djls-workspace/src/lib.rs create mode 100644 crates/djls-workspace/src/vfs.rs diff --git a/Cargo.lock b/Cargo.lock index b48cdd9..95b7a76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,6 +168,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "camino" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" + [[package]] name = "cfg-if" version = "1.0.1" @@ -403,6 +409,17 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "djls" version = "5.2.0-alpha" @@ -453,10 +470,13 @@ name = "djls-server" version = "0.0.0" dependencies = [ "anyhow", + "camino", + "dashmap", "djls-conf", "djls-dev", "djls-project", "djls-templates", + "djls-workspace", "percent-encoding", "pyo3", "salsa", @@ -467,6 +487,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "url", ] [[package]] @@ -481,6 +502,16 @@ dependencies = [ "toml", ] +[[package]] +name = "djls-workspace" +version = "0.0.0" +dependencies = [ + "camino", + "dashmap", + "salsa", + "url", +] + [[package]] name = "dlv-list" version = "0.5.2" @@ -564,6 +595,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures" version = "0.3.31" @@ -718,6 +758,113 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.10.0" @@ -817,6 +964,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + [[package]] name = "lock_api" version = "0.4.13" @@ -1056,6 +1209,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -1465,6 +1627,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strsim" version = "0.11.1" @@ -1613,6 +1781,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tokio" version = "1.47.1" @@ -1859,6 +2037,23 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -2111,6 +2306,12 @@ dependencies = [ "bitflags 2.9.2", ] +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + [[package]] name = "yaml-rust2" version = "0.10.3" @@ -2121,3 +2322,81 @@ dependencies = [ "encoding_rs", "hashlink", ] + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 2a674a3..8a88a0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ djls-dev = { path = "crates/djls-dev" } djls-project = { path = "crates/djls-project" } djls-server = { path = "crates/djls-server" } djls-templates = { path = "crates/djls-templates" } +djls-workspace = { path = "crates/djls-workspace" } # core deps, pin exact versions pyo3 = "0.25.0" @@ -17,8 +18,10 @@ salsa = "0.23.0" tower-lsp-server = { version = "0.22.0", features = ["proposed"] } anyhow = "1.0" +camino = "1.1" clap = { version = "4.5", features = ["derive"] } config = { version ="0.15", features = ["toml"] } +dashmap = "6.1" directories = "6.0" percent-encoding = "2.3" serde = { version = "1.0", features = ["derive"] } @@ -29,6 +32,7 @@ toml = "0.9" tracing = "0.1" tracing-appender = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt", "time"] } +url = "2.5" which = "8.0" # testing diff --git a/crates/djls-server/Cargo.toml b/crates/djls-server/Cargo.toml index 396f0f3..e3f27f8 100644 --- a/crates/djls-server/Cargo.toml +++ b/crates/djls-server/Cargo.toml @@ -11,8 +11,11 @@ default = [] djls-conf = { workspace = true } djls-project = { workspace = true } djls-templates = { workspace = true } +djls-workspace = { workspace = true } anyhow = { workspace = true } +camino = { workspace = true } +dashmap = { workspace = true } percent-encoding = { workspace = true } pyo3 = { workspace = true } salsa = { workspace = true } @@ -23,6 +26,7 @@ tower-lsp-server = { workspace = true } tracing = { workspace = true } tracing-appender = { workspace = true } tracing-subscriber = { workspace = true } +url = { workspace = true } [build-dependencies] djls-dev = { workspace = true } diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml new file mode 100644 index 0000000..9f8edce --- /dev/null +++ b/crates/djls-workspace/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "djls-workspace" +version = "0.0.0" +edition = "2021" + +[dependencies] +camino = { workspace = true } +dashmap = { workspace = true } +salsa = { workspace = true } +url = { workspace = true } + +[dev-dependencies] + +[lints] +workspace = true diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs new file mode 100644 index 0000000..97f894e --- /dev/null +++ b/crates/djls-workspace/src/bridge.rs @@ -0,0 +1,108 @@ +//! Bridge between VFS snapshots and Salsa inputs. +//! +//! The bridge module isolates Salsa input mutation behind a single, idempotent API. +//! It ensures we only touch Salsa when content or classification changes, maximizing +//! incremental performance. + +use std::{collections::HashMap, sync::Arc}; + +use salsa::Setter; + +use super::{ + db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}, + vfs::{FileKind, VfsSnapshot}, + FileId, +}; + +/// Owner of the Salsa [`Database`] plus the handles for updating inputs. +/// +/// [`FileStore`] serves as the bridge between the VFS (with [`FileId`]s) and Salsa (with entities). +/// It maintains a mapping from [`FileId`]s to [`SourceFile`] entities and manages the global +/// [`TemplateLoaderOrder`] input. The [`FileStore`] ensures that Salsa inputs are only mutated +/// when actual changes occur, preserving incremental computation efficiency. +pub struct FileStore { + /// The Salsa DB instance + pub db: Database, + /// Map from [`FileId`] to its Salsa input entity + files: HashMap, + /// Handle to the global template loader configuration input + template_loader: Option, +} + +impl FileStore { + /// Construct an empty store and DB. + pub fn new() -> Self { + Self { + db: Database::default(), + files: HashMap::new(), + template_loader: None, + } + } + + /// Create or update the global template loader order input. + /// + /// Sets the ordered list of template root directories that Django will search + /// when resolving template names. If the input already exists, it updates the + /// existing value; otherwise, it creates a new [`TemplateLoaderOrder`] input. + pub fn set_template_loader_order(&mut self, ordered_roots: Vec) { + let roots = Arc::from(ordered_roots.into_boxed_slice()); + if let Some(tl) = self.template_loader { + tl.set_roots(&mut self.db).to(roots); + } else { + self.template_loader = Some(TemplateLoaderOrder::new(&self.db, roots)); + } + } + + /// Mirror a VFS snapshot into Salsa inputs. + /// + /// This method is the core synchronization point between the VFS and Salsa. + /// It iterates through all files in the snapshot and: + /// - Creates [`SourceFile`] inputs for new files + /// - Updates `.text` and `.kind` only when changed to preserve incremental reuse + /// + /// The method is idempotent and minimizes Salsa invalidations by checking for + /// actual changes before updating inputs. + pub fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { + for (id, rec) in &snap.files { + let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); + let new_kind = match rec.meta.kind { + FileKind::Python => FileKindMini::Python, + FileKind::Template => FileKindMini::Template, + FileKind::Other => FileKindMini::Other, + }; + + if let Some(sf) = self.files.get(id) { + // Update if changed — avoid touching Salsa when not needed + if sf.kind(&self.db) != new_kind { + sf.set_kind(&mut self.db).to(new_kind.clone()); + } + if sf.text(&self.db).as_ref() != &*new_text { + sf.set_text(&mut self.db).to(new_text.clone()); + } + } else { + let sf = SourceFile::new(&self.db, new_kind, new_text); + self.files.insert(*id, sf); + } + } + } + + /// Get the text content of a file by its [`FileId`]. + /// + /// Returns `None` if the file is not tracked in the [`FileStore`]. + pub fn file_text(&self, id: FileId) -> Option> { + self.files.get(&id).map(|sf| sf.text(&self.db).clone()) + } + + /// Get the file kind classification by its [`FileId`]. + /// + /// Returns `None` if the file is not tracked in the [`FileStore`]. + pub fn file_kind(&self, id: FileId) -> Option { + self.files.get(&id).map(|sf| sf.kind(&self.db)) + } +} + +impl Default for FileStore { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs new file mode 100644 index 0000000..91aa1cc --- /dev/null +++ b/crates/djls-workspace/src/db.rs @@ -0,0 +1,91 @@ +//! Salsa database and input entities for workspace. +//! +//! This module defines the Salsa world—what can be set and tracked incrementally. +//! Inputs are kept minimal to avoid unnecessary recomputation. + +use std::sync::Arc; +#[cfg(test)] +use std::sync::Mutex; + +/// Salsa database root for workspace +/// +/// The [`Database`] provides default storage and, in tests, captures Salsa events for +/// reuse/diagnostics. It serves as the core incremental computation engine, tracking +/// dependencies and invalidations across all inputs and derived queries. +#[salsa::db] +#[derive(Clone)] +#[cfg_attr(not(test), derive(Default))] +pub struct Database { + storage: salsa::Storage, + + // The logs are only used for testing and demonstrating reuse: + #[cfg(test)] + logs: Arc>>>, +} + +#[cfg(test)] +impl Default for Database { + fn default() -> Self { + let logs = >>>>::default(); + Self { + storage: salsa::Storage::new(Some(Box::new({ + let logs = logs.clone(); + move |event| { + eprintln!("Event: {event:?}"); + // Log interesting events, if logging is enabled + if let Some(logs) = &mut *logs.lock().unwrap() { + // only log interesting events + if let salsa::EventKind::WillExecute { .. } = event.kind { + logs.push(format!("Event: {event:?}")); + } + } + } + }))), + logs, + } + } +} + +#[salsa::db] +impl salsa::Database for Database {} + +/// Minimal classification for analysis routing. +/// +/// [`FileKindMini`] provides a lightweight categorization of files to determine which +/// analysis pipelines should process them. This is the Salsa-side representation +/// of file types, mapped from the VFS layer's `vfs::FileKind`. +#[derive(Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKindMini { + /// Python source file (.py) + Python, + /// Django template file (.html, .jinja, etc.) + Template, + /// Other file types not requiring specialized analysis + Other, +} + +/// Represents a single file's classification and current content. +/// +/// [`SourceFile`] is a Salsa input entity that tracks both the file's type (for routing +/// to appropriate analyzers) and its current text content. The text is stored as +/// `Arc` for efficient sharing across the incremental computation graph. +#[salsa::input] +pub struct SourceFile { + /// The file's classification for analysis routing + pub kind: FileKindMini, + /// The current text content of the file + #[returns(ref)] + pub text: Arc, +} + +/// Global input configuring ordered template loader roots. +/// +/// [`TemplateLoaderOrder`] represents the Django `TEMPLATES[n]['DIRS']` configuration, +/// defining the search order for template resolution. This is a global input that +/// affects template name resolution across the entire project. +#[salsa::input] +pub struct TemplateLoaderOrder { + /// Ordered list of template root directories + #[returns(ref)] + pub roots: Arc<[String]>, +} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs new file mode 100644 index 0000000..aea4a53 --- /dev/null +++ b/crates/djls-workspace/src/lib.rs @@ -0,0 +1,30 @@ +mod bridge; +mod db; +mod vfs; + +// Re-export public API +pub use bridge::FileStore; +pub use db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}; +pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; + +/// Stable, compact identifier for files across the subsystem. +/// +/// [`FileId`] decouples file identity from paths/URIs, providing efficient keys for maps and +/// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the +/// lifetime of the VFS, even if the file's content or metadata changes. +#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] +pub struct FileId(u32); + +impl FileId { + /// Create a [`FileId`] from a raw u32 value. + #[must_use] + pub fn from_raw(raw: u32) -> Self { + FileId(raw) + } + + /// Get the underlying u32 index value. + #[must_use] + pub fn index(self) -> u32 { + self.0 + } +} diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs new file mode 100644 index 0000000..65f4008 --- /dev/null +++ b/crates/djls-workspace/src/vfs.rs @@ -0,0 +1,245 @@ +//! Change-tracked, concurrent virtual file system keyed by [`FileId`]. +//! +//! The VFS provides thread-safe, identity-stable storage with cheap change detection +//! and snapshotting. Downstream systems consume snapshots to avoid locking and to +//! batch updates. + +use camino::Utf8PathBuf; +use dashmap::DashMap; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU32, AtomicU64, Ordering}, + Arc, + }, +}; +use url::Url; + +use super::FileId; + +/// Monotonic counter representing global VFS state. +/// +/// [`Revision`] increments whenever file content changes occur in the VFS. +/// This provides a cheap way to detect if any changes have occurred since +/// a previous snapshot was taken. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default, PartialOrd, Ord)] +pub struct Revision(u64); + +impl Revision { + /// Create a [`Revision`] from a raw u64 value. + pub fn from_raw(raw: u64) -> Self { + Revision(raw) + } + + /// Get the underlying u64 value. + pub fn value(self) -> u64 { + self.0 + } +} + +/// File classification at the VFS layer. +/// +/// [`FileKind`] determines how a file should be processed by downstream analyzers. +/// This classification is performed when files are first ingested into the VFS. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKind { + /// Python source file + Python, + /// Django template file + Template, + /// Other file type + Other, +} + +/// Metadata associated with a file in the VFS. +/// +/// [`FileMeta`] contains all non-content information about a file, including its +/// identity (URI), filesystem path, classification, and optional version number +/// from the LSP client. +#[derive(Clone, Debug)] +pub struct FileMeta { + /// The file's URI (typically file:// scheme) + pub uri: Url, + /// The file's path in the filesystem + pub path: Utf8PathBuf, + /// Classification for routing to analyzers + pub kind: FileKind, + /// Optional LSP document version + pub version: Option, +} + +/// Source of text content in the VFS. +/// +/// [`TextSource`] tracks where file content originated from, which is useful for +/// debugging and understanding the current state of the VFS. All variants hold +/// `Arc` for efficient sharing. +#[derive(Clone)] +pub enum TextSource { + /// Content loaded from disk + Disk(Arc), + /// Content from LSP client overlay (in-memory edits) + Overlay(Arc), + /// Content generated programmatically + Generated(Arc), +} + +/// Complete record of a file in the VFS. +/// +/// [`FileRecord`] combines metadata, current text content, and a content hash +/// for efficient change detection. +#[derive(Clone)] +pub struct FileRecord { + /// File metadata (URI, path, kind, version) + pub meta: FileMeta, + /// Current text content and its source + pub text: TextSource, + /// Hash of current content for change detection + pub hash: u64, +} + +/// Thread-safe virtual file system with change tracking. +/// +/// [`Vfs`] provides concurrent access to file content with stable [`FileId`] assignment, +/// content hashing for change detection, and atomic snapshot generation. It uses +/// `DashMap` for lock-free concurrent access and atomic counters for revision tracking. +pub struct Vfs { + /// Atomic counter for generating unique [`FileId`]s + next_file_id: AtomicU32, + /// Map from URI to [`FileId`] for deduplication + by_uri: DashMap, + /// Map from [`FileId`] to [`FileRecord`] for content storage + files: DashMap, + /// Global revision counter, incremented on content changes + head: AtomicU64, +} + +impl Vfs { + /// Construct an empty VFS. + pub fn new() -> Self { + Self { + next_file_id: AtomicU32::new(0), + by_uri: DashMap::new(), + files: DashMap::new(), + head: AtomicU64::new(0), + } + } + + /// Get or create a [`FileId`] for the given URI. + /// + /// Returns the existing [`FileId`] if the URI is already known, or creates a new + /// [`FileRecord`] with the provided metadata and text. This method computes and + /// stores a content hash for change detection. + pub fn intern_file( + &self, + uri: Url, + path: Utf8PathBuf, + kind: FileKind, + text: TextSource, + ) -> FileId { + if let Some(id) = self.by_uri.get(&uri).map(|entry| *entry) { + return id; + } + let id = FileId(self.next_file_id.fetch_add(1, Ordering::SeqCst)); + let meta = FileMeta { + uri: uri.clone(), + path, + kind, + version: None, + }; + let hash = content_hash(&text); + self.by_uri.insert(uri, id); + self.files.insert(id, FileRecord { meta, text, hash }); + id + } + + /// Set overlay text for a file, typically from LSP didChange events. + /// + /// Updates the file's text to an Overlay variant with the new content. + /// Only increments the global revision if the content actually changed + /// (detected via hash comparison). + /// + /// Returns a tuple of (new global revision, whether content changed). + pub fn set_overlay( + &self, + id: FileId, + version: Option, + new_text: Arc, + ) -> (Revision, bool) { + let mut rec = self.files.get_mut(&id).expect("unknown file"); + rec.meta.version = version; + let next = TextSource::Overlay(new_text); + let new_hash = content_hash(&next); + let changed = new_hash != rec.hash; + if changed { + rec.text = next; + rec.hash = new_hash; + self.head.fetch_add(1, Ordering::SeqCst); + } + ( + Revision::from_raw(self.head.load(Ordering::SeqCst)), + changed, + ) + } + + /// Create an immutable snapshot of the current VFS state. + /// + /// Materializes a consistent view of all files for downstream consumers. + /// The snapshot includes the current revision and a clone of all file records. + /// This operation is relatively cheap due to `Arc` sharing of text content. + pub fn snapshot(&self) -> VfsSnapshot { + VfsSnapshot { + revision: Revision::from_raw(self.head.load(Ordering::SeqCst)), + files: self + .files + .iter() + .map(|entry| (*entry.key(), entry.value().clone())) + .collect(), + } + } +} + +/// Compute a stable hash over file content. +/// +/// Used for efficient change detection - if the hash hasn't changed, +/// the content hasn't changed, avoiding unnecessary Salsa invalidations. +fn content_hash(src: &TextSource) -> u64 { + let s: &str = match src { + TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, + }; + let mut h = DefaultHasher::new(); + s.hash(&mut h); + h.finish() +} + +/// Immutable snapshot view of the VFS at a specific revision. +/// +/// [`VfsSnapshot`] provides a consistent view of all files for downstream consumers, +/// avoiding the need for locking during processing. Snapshots are created atomically +/// and can be safely shared across threads. +#[derive(Clone)] +pub struct VfsSnapshot { + /// The global revision at the time of snapshot + pub revision: Revision, + /// All files in the VFS at snapshot time + pub files: HashMap, +} + +impl VfsSnapshot { + /// Get the text content of a file in this snapshot. + /// + /// Returns `None` if the [`FileId`] is not present in the snapshot. + pub fn get_text(&self, id: FileId) -> Option> { + self.files.get(&id).map(|r| match &r.text { + TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s.clone(), + }) + } + + /// Get the metadata for a file in this snapshot. + /// + /// Returns `None` if the [`FileId`] is not present in the snapshot. + pub fn meta(&self, id: FileId) -> Option<&FileMeta> { + self.files.get(&id).map(|r| &r.meta) + } +} From 74b6b5b56daedfdcf79e632194c8512062c3bd50 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 03:47:56 -0500 Subject: [PATCH 02/30] wip --- Cargo.lock | 1 + crates/djls-server/src/server.rs | 10 +- crates/djls-server/src/workspace/document.rs | 154 ++++++--------- crates/djls-server/src/workspace/store.rs | 195 +++++++++++++------ crates/djls-workspace/Cargo.toml | 1 + crates/djls-workspace/src/vfs.rs | 53 ++--- 6 files changed, 230 insertions(+), 184 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95b7a76..48cf438 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,6 +506,7 @@ dependencies = [ name = "djls-workspace" version = "0.0.0" dependencies = [ + "anyhow", "camino", "dashmap", "salsa", diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 3977ef6..9df7c63 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -218,8 +218,9 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let db = session.db(); - session.documents_mut().handle_did_open(&db, ¶ms); + if let Err(e) = session.documents_mut().handle_did_open(¶ms) { + tracing::error!("Failed to handle did_open: {}", e); + } }) .await; } @@ -228,8 +229,7 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let db = session.db(); - let _ = session.documents_mut().handle_did_change(&db, ¶ms); + let _ = session.documents_mut().handle_did_change(¶ms); }) .await; } @@ -248,9 +248,7 @@ impl LanguageServer for DjangoLanguageServer { .with_session(|session| { if let Some(project) = session.project() { if let Some(tags) = project.template_tags() { - let db = session.db(); return session.documents().get_completions( - &db, params.text_document_position.text_document.uri.as_str(), params.text_document_position.position, tags, diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs index 4c23f13..3078873 100644 --- a/crates/djls-server/src/workspace/document.rs +++ b/crates/djls-server/src/workspace/document.rs @@ -1,113 +1,66 @@ -use salsa::Database; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; +use std::sync::Arc; +use tower_lsp_server::lsp_types::{Position, Range}; +use djls_workspace::{FileId, VfsSnapshot}; -#[salsa::input(debug)] +/// Document metadata container - no longer a Salsa input, just plain data +#[derive(Clone, Debug)] pub struct TextDocument { - #[returns(ref)] pub uri: String, - #[returns(ref)] - pub contents: String, - #[returns(ref)] - pub index: LineIndex, pub version: i32, pub language_id: LanguageId, + file_id: FileId, } impl TextDocument { - pub fn from_did_open_params(db: &dyn Database, params: &DidOpenTextDocumentParams) -> Self { - let uri = params.text_document.uri.to_string(); - let contents = params.text_document.text.clone(); - let version = params.text_document.version; - let language_id = LanguageId::from(params.text_document.language_id.as_str()); - - let index = LineIndex::new(&contents); - TextDocument::new(db, uri, contents, index, version, language_id) - } - - pub fn with_changes( - self, - db: &dyn Database, - changes: &[TextDocumentContentChangeEvent], - new_version: i32, - ) -> Self { - let mut new_contents = self.contents(db).to_string(); - - for change in changes { - if let Some(range) = change.range { - let index = LineIndex::new(&new_contents); - - if let (Some(start_offset), Some(end_offset)) = ( - index.offset(range.start).map(|o| o as usize), - index.offset(range.end).map(|o| o as usize), - ) { - let mut updated_content = String::with_capacity( - new_contents.len() - (end_offset - start_offset) + change.text.len(), - ); - - updated_content.push_str(&new_contents[..start_offset]); - updated_content.push_str(&change.text); - updated_content.push_str(&new_contents[end_offset..]); - - new_contents = updated_content; - } - } else { - // Full document update - new_contents.clone_from(&change.text); - } + pub fn new(uri: String, version: i32, language_id: LanguageId, file_id: FileId) -> Self { + Self { + uri, + version, + language_id, + file_id, } - - let index = LineIndex::new(&new_contents); - TextDocument::new( - db, - self.uri(db).to_string(), - new_contents, - index, - new_version, - self.language_id(db), - ) } - - #[allow(dead_code)] - pub fn get_text(self, db: &dyn Database) -> String { - self.contents(db).to_string() + + pub fn file_id(&self) -> FileId { + self.file_id } - - #[allow(dead_code)] - pub fn get_text_range(self, db: &dyn Database, range: Range) -> Option { - let index = self.index(db); - let start = index.offset(range.start)? as usize; - let end = index.offset(range.end)? as usize; - let contents = self.contents(db); - Some(contents[start..end].to_string()) + + pub fn get_content(&self, vfs: &VfsSnapshot) -> Option> { + vfs.get_text(self.file_id) } - - pub fn get_line(self, db: &dyn Database, line: u32) -> Option { - let index = self.index(db); - let start = index.line_starts.get(line as usize)?; - let end = index - .line_starts + + pub fn get_line(&self, vfs: &VfsSnapshot, line_index: &LineIndex, line: u32) -> Option { + let content = self.get_content(vfs)?; + + let line_start = *line_index.line_starts.get(line as usize)?; + let line_end = line_index.line_starts .get(line as usize + 1) .copied() - .unwrap_or(index.length); - - let contents = self.contents(db); - Some(contents[*start as usize..end as usize].to_string()) + .unwrap_or(line_index.length); + + Some(content[line_start as usize..line_end as usize].to_string()) } - - #[allow(dead_code)] - pub fn line_count(self, db: &dyn Database) -> usize { - self.index(db).line_starts.len() + + pub fn get_text_range(&self, vfs: &VfsSnapshot, line_index: &LineIndex, range: Range) -> Option { + let content = self.get_content(vfs)?; + + let start_offset = line_index.offset(range.start)? as usize; + let end_offset = line_index.offset(range.end)? as usize; + + Some(content[start_offset..end_offset].to_string()) } + + pub fn get_template_tag_context(&self, vfs: &VfsSnapshot, line_index: &LineIndex, position: Position) -> Option { + let content = self.get_content(vfs)?; + + let start = line_index.line_starts.get(position.line as usize)?; + let end = line_index + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(line_index.length); - pub fn get_template_tag_context( - self, - db: &dyn Database, - position: Position, - ) -> Option { - let line = self.get_line(db, position.line)?; + let line = &content[*start as usize..end as usize]; let char_pos: usize = position.character.try_into().ok()?; let prefix = &line[..char_pos]; let rest_of_line = &line[char_pos..]; @@ -136,8 +89,8 @@ impl TextDocument { #[derive(Clone, Debug)] pub struct LineIndex { - line_starts: Vec, - length: u32, + pub line_starts: Vec, + pub length: u32, } impl LineIndex { @@ -201,6 +154,16 @@ impl From for LanguageId { } } +impl From for djls_workspace::FileKind { + fn from(language_id: LanguageId) -> Self { + match language_id { + LanguageId::Python => Self::Python, + LanguageId::HtmlDjango => Self::Template, + LanguageId::Other => Self::Other, + } + } +} + #[derive(Debug)] pub enum ClosingBrace { None, @@ -214,3 +177,4 @@ pub struct TemplateTagContext { pub closing_brace: ClosingBrace, pub needs_leading_space: bool, } + diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 3ec2109..0026fd0 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -1,9 +1,11 @@ use std::collections::HashMap; +use std::sync::Arc; use anyhow::anyhow; use anyhow::Result; +use camino::Utf8PathBuf; use djls_project::TemplateTags; -use salsa::Database; +use djls_workspace::{FileId, FileKind, TextSource, Vfs}; use tower_lsp_server::lsp_types::CompletionItem; use tower_lsp_server::lsp_types::CompletionItemKind; use tower_lsp_server::lsp_types::CompletionResponse; @@ -16,83 +18,149 @@ use tower_lsp_server::lsp_types::MarkupContent; use tower_lsp_server::lsp_types::MarkupKind; use tower_lsp_server::lsp_types::Position; -use super::document::ClosingBrace; -use super::document::LanguageId; -use super::document::TextDocument; +use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; -#[derive(Debug, Default)] pub struct Store { - documents: HashMap, + vfs: Arc, + file_ids: HashMap, + line_indices: HashMap, versions: HashMap, + documents: HashMap, +} + +impl Default for Store { + fn default() -> Self { + Self { + vfs: Arc::new(Vfs::default()), + file_ids: HashMap::new(), + line_indices: HashMap::new(), + versions: HashMap::new(), + documents: HashMap::new(), + } + } } impl Store { - pub fn handle_did_open(&mut self, db: &dyn Database, params: &DidOpenTextDocumentParams) { - let uri = params.text_document.uri.to_string(); + pub fn handle_did_open(&mut self, params: &DidOpenTextDocumentParams) -> Result<()> { + let uri_str = params.text_document.uri.to_string(); + let uri = params.text_document.uri.clone(); let version = params.text_document.version; + let content = params.text_document.text.clone(); + let language_id = LanguageId::from(params.text_document.language_id.as_str()); + let kind = FileKind::from(language_id.clone()); - let document = TextDocument::from_did_open_params(db, params); + // Convert URI to Url for VFS + let vfs_url = + url::Url::parse(&uri.to_string()).map_err(|e| anyhow!("Invalid URI: {}", e))?; - self.add_document(document, uri.clone()); - self.versions.insert(uri, version); + // Convert to path - simplified for now, just use URI string + let path = Utf8PathBuf::from(uri.as_str()); + + // Store content in VFS + let text_source = TextSource::Overlay(Arc::from(content.as_str())); + let file_id = self.vfs.intern_file(vfs_url, path, kind, text_source); + + // Set overlay content in VFS + self.vfs.set_overlay(file_id, Arc::from(content.as_str()))?; + + // Create TextDocument metadata + let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); + self.documents.insert(uri_str.clone(), document); + + // Cache mappings and indices + self.file_ids.insert(uri_str.clone(), file_id); + self.line_indices.insert(file_id, LineIndex::new(&content)); + self.versions.insert(uri_str, version); + + Ok(()) } - pub fn handle_did_change( - &mut self, - db: &dyn Database, - params: &DidChangeTextDocumentParams, - ) -> Result<()> { - let uri = params.text_document.uri.as_str().to_string(); + pub fn handle_did_change(&mut self, params: &DidChangeTextDocumentParams) -> Result<()> { + let uri_str = params.text_document.uri.as_str().to_string(); let version = params.text_document.version; - let document = self - .get_document(&uri) - .ok_or_else(|| anyhow!("Document not found: {}", uri))?; + // Look up FileId + let file_id = self + .file_ids + .get(&uri_str) + .copied() + .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; - let new_document = document.with_changes(db, ¶ms.content_changes, version); + // Get current content from VFS + let snapshot = self.vfs.snapshot(); + let current_content = snapshot + .get_text(file_id) + .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - self.documents.insert(uri.clone(), new_document); - self.versions.insert(uri, version); + // Apply text changes + let mut new_content = current_content.to_string(); + for change in ¶ms.content_changes { + if let Some(range) = change.range { + // Get current line index for position calculations + let line_index = self + .line_indices + .get(&file_id) + .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; + + if let (Some(start_offset), Some(end_offset)) = ( + line_index.offset(range.start).map(|o| o as usize), + line_index.offset(range.end).map(|o| o as usize), + ) { + let mut updated_content = String::with_capacity( + new_content.len() - (end_offset - start_offset) + change.text.len(), + ); + + updated_content.push_str(&new_content[..start_offset]); + updated_content.push_str(&change.text); + updated_content.push_str(&new_content[end_offset..]); + + new_content = updated_content; + } + } else { + // Full document update + new_content.clone_from(&change.text); + } + } + + // Update TextDocument version + if let Some(document) = self.documents.get_mut(&uri_str) { + document.version = version; + } + + // Update VFS with new content + self.vfs + .set_overlay(file_id, Arc::from(new_content.as_str()))?; + + // Update cached line index and version + self.line_indices + .insert(file_id, LineIndex::new(&new_content)); + self.versions.insert(uri_str, version); Ok(()) } pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { - self.remove_document(params.text_document.uri.as_str()); + let uri_str = params.text_document.uri.as_str(); + + // Remove TextDocument metadata + self.documents.remove(uri_str); + + // Look up FileId and remove mappings + if let Some(file_id) = self.file_ids.remove(uri_str) { + self.line_indices.remove(&file_id); + } + self.versions.remove(uri_str); + + // Note: We don't remove from VFS as it might be useful for caching + // The VFS will handle cleanup internally } - fn add_document(&mut self, document: TextDocument, uri: String) { - self.documents.insert(uri, document); + pub fn get_file_id(&self, uri: &str) -> Option { + self.file_ids.get(uri).copied() } - fn remove_document(&mut self, uri: &str) { - self.documents.remove(uri); - self.versions.remove(uri); - } - - fn get_document(&self, uri: &str) -> Option<&TextDocument> { - self.documents.get(uri) - } - - #[allow(dead_code)] - fn get_document_mut(&mut self, uri: &str) -> Option<&mut TextDocument> { - self.documents.get_mut(uri) - } - - #[allow(dead_code)] - pub fn get_all_documents(&self) -> impl Iterator { - self.documents.values() - } - - #[allow(dead_code)] - pub fn get_documents_by_language<'db>( - &'db self, - db: &'db dyn Database, - language_id: LanguageId, - ) -> impl Iterator + 'db { - self.documents - .values() - .filter(move |doc| doc.language_id(db) == language_id) + pub fn get_line_index(&self, file_id: FileId) -> Option<&LineIndex> { + self.line_indices.get(&file_id) } #[allow(dead_code)] @@ -105,20 +173,31 @@ impl Store { self.get_version(uri) == Some(version) } + // TextDocument helper methods + pub fn get_document(&self, uri: &str) -> Option<&TextDocument> { + self.documents.get(uri) + } + + pub fn get_document_mut(&mut self, uri: &str) -> Option<&mut TextDocument> { + self.documents.get_mut(uri) + } + pub fn get_completions( &self, - db: &dyn Database, uri: &str, position: Position, tags: &TemplateTags, ) -> Option { + // Check if this is a Django template using TextDocument metadata let document = self.get_document(uri)?; - - if document.language_id(db) != LanguageId::HtmlDjango { + if document.language_id != LanguageId::HtmlDjango { return None; } - let context = document.get_template_tag_context(db, position)?; + // Get template tag context from document + let vfs_snapshot = self.vfs.snapshot(); + let line_index = self.get_line_index(document.file_id())?; + let context = document.get_template_tag_context(&vfs_snapshot, line_index, position)?; let mut completions: Vec = tags .iter() diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 9f8edce..9b5ee6c 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -4,6 +4,7 @@ version = "0.0.0" edition = "2021" [dependencies] +anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } salsa = { workspace = true } diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs index 65f4008..9ebe6a2 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs.rs @@ -4,6 +4,7 @@ //! and snapshotting. Downstream systems consume snapshots to avoid locking and to //! batch updates. +use anyhow::{anyhow, Result}; use camino::Utf8PathBuf; use dashmap::DashMap; use std::collections::hash_map::DefaultHasher; @@ -29,11 +30,13 @@ pub struct Revision(u64); impl Revision { /// Create a [`Revision`] from a raw u64 value. + #[must_use] pub fn from_raw(raw: u64) -> Self { Revision(raw) } /// Get the underlying u64 value. + #[must_use] pub fn value(self) -> u64 { self.0 } @@ -56,8 +59,7 @@ pub enum FileKind { /// Metadata associated with a file in the VFS. /// /// [`FileMeta`] contains all non-content information about a file, including its -/// identity (URI), filesystem path, classification, and optional version number -/// from the LSP client. +/// identity (URI), filesystem path, and classification. #[derive(Clone, Debug)] pub struct FileMeta { /// The file's URI (typically file:// scheme) @@ -66,8 +68,6 @@ pub struct FileMeta { pub path: Utf8PathBuf, /// Classification for routing to analyzers pub kind: FileKind, - /// Optional LSP document version - pub version: Option, } /// Source of text content in the VFS. @@ -116,16 +116,6 @@ pub struct Vfs { } impl Vfs { - /// Construct an empty VFS. - pub fn new() -> Self { - Self { - next_file_id: AtomicU32::new(0), - by_uri: DashMap::new(), - files: DashMap::new(), - head: AtomicU64::new(0), - } - } - /// Get or create a [`FileId`] for the given URI. /// /// Returns the existing [`FileId`] if the URI is already known, or creates a new @@ -146,7 +136,6 @@ impl Vfs { uri: uri.clone(), path, kind, - version: None, }; let hash = content_hash(&text); self.by_uri.insert(uri, id); @@ -161,14 +150,15 @@ impl Vfs { /// (detected via hash comparison). /// /// Returns a tuple of (new global revision, whether content changed). - pub fn set_overlay( - &self, - id: FileId, - version: Option, - new_text: Arc, - ) -> (Revision, bool) { - let mut rec = self.files.get_mut(&id).expect("unknown file"); - rec.meta.version = version; + /// + /// # Errors + /// + /// Returns an error if the provided `FileId` does not exist in the VFS. + pub fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { + let mut rec = self + .files + .get_mut(&id) + .ok_or_else(|| anyhow!("unknown file: {:?}", id))?; let next = TextSource::Overlay(new_text); let new_hash = content_hash(&next); let changed = new_hash != rec.hash; @@ -177,10 +167,10 @@ impl Vfs { rec.hash = new_hash; self.head.fetch_add(1, Ordering::SeqCst); } - ( + Ok(( Revision::from_raw(self.head.load(Ordering::SeqCst)), changed, - ) + )) } /// Create an immutable snapshot of the current VFS state. @@ -200,6 +190,17 @@ impl Vfs { } } +impl Default for Vfs { + fn default() -> Self { + Self { + next_file_id: AtomicU32::new(0), + by_uri: DashMap::new(), + files: DashMap::new(), + head: AtomicU64::new(0), + } + } +} + /// Compute a stable hash over file content. /// /// Used for efficient change detection - if the hash hasn't changed, @@ -230,6 +231,7 @@ impl VfsSnapshot { /// Get the text content of a file in this snapshot. /// /// Returns `None` if the [`FileId`] is not present in the snapshot. + #[must_use] pub fn get_text(&self, id: FileId) -> Option> { self.files.get(&id).map(|r| match &r.text { TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s.clone(), @@ -239,6 +241,7 @@ impl VfsSnapshot { /// Get the metadata for a file in this snapshot. /// /// Returns `None` if the [`FileId`] is not present in the snapshot. + #[must_use] pub fn meta(&self, id: FileId) -> Option<&FileMeta> { self.files.get(&id).map(|r| &r.meta) } From 48dacb277c9f54d7b72d79732b5bb032aacebc49 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 04:25:35 -0500 Subject: [PATCH 03/30] wip --- crates/djls-server/src/workspace/document.rs | 48 +++- crates/djls-server/src/workspace/store.rs | 274 +++++++++++++++++-- 2 files changed, 290 insertions(+), 32 deletions(-) diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs index 3078873..5a77a65 100644 --- a/crates/djls-server/src/workspace/document.rs +++ b/crates/djls-server/src/workspace/document.rs @@ -90,24 +90,32 @@ impl TextDocument { #[derive(Clone, Debug)] pub struct LineIndex { pub line_starts: Vec, + pub line_starts_utf16: Vec, pub length: u32, + pub length_utf16: u32, } impl LineIndex { pub fn new(text: &str) -> Self { let mut line_starts = vec![0]; - let mut pos = 0; + let mut line_starts_utf16 = vec![0]; + let mut pos_utf8 = 0; + let mut pos_utf16 = 0; for c in text.chars() { - pos += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); if c == '\n' { - line_starts.push(pos); + line_starts.push(pos_utf8); + line_starts_utf16.push(pos_utf16); } } Self { line_starts, - length: pos, + line_starts_utf16, + length: pos_utf8, + length_utf16: pos_utf16, } } @@ -117,6 +125,38 @@ impl LineIndex { Some(line_start + position.character) } + /// Convert UTF-16 LSP position to UTF-8 byte offset + pub fn offset_utf16(&self, position: Position, text: &str) -> Option { + let line_start_utf8 = self.line_starts.get(position.line as usize)?; + let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + + // If position is at start of line, return UTF-8 line start + if position.character == 0 { + return Some(*line_start_utf8); + } + + // Find the line text + let next_line_start = self.line_starts.get(position.line as usize + 1) + .copied() + .unwrap_or(self.length); + + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + + // Convert UTF-16 character offset to UTF-8 byte offset within the line + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); + } + + Some(line_start_utf8 + utf8_pos) + } + #[allow(dead_code)] pub fn position(&self, offset: u32) -> Position { let line = match self.line_starts.binary_search(&offset) { diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 0026fd0..1a9efbb 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -12,6 +12,7 @@ use tower_lsp_server::lsp_types::CompletionResponse; use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use tower_lsp_server::lsp_types::Documentation; use tower_lsp_server::lsp_types::InsertTextFormat; use tower_lsp_server::lsp_types::MarkupContent; @@ -92,35 +93,14 @@ impl Store { .get_text(file_id) .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - // Apply text changes - let mut new_content = current_content.to_string(); - for change in ¶ms.content_changes { - if let Some(range) = change.range { - // Get current line index for position calculations - let line_index = self - .line_indices - .get(&file_id) - .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; + // Get current line index for position calculations + let line_index = self + .line_indices + .get(&file_id) + .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; - if let (Some(start_offset), Some(end_offset)) = ( - line_index.offset(range.start).map(|o| o as usize), - line_index.offset(range.end).map(|o| o as usize), - ) { - let mut updated_content = String::with_capacity( - new_content.len() - (end_offset - start_offset) + change.text.len(), - ); - - updated_content.push_str(&new_content[..start_offset]); - updated_content.push_str(&change.text); - updated_content.push_str(&new_content[end_offset..]); - - new_content = updated_content; - } - } else { - // Full document update - new_content.clone_from(&change.text); - } - } + // Apply text changes using the new function + let new_content = apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; // Update TextDocument version if let Some(document) = self.documents.get_mut(&uri_str) { @@ -235,3 +215,241 @@ impl Store { } } } + +/// Apply text changes to content, handling multiple changes correctly +fn apply_text_changes( + content: &str, + changes: &[TextDocumentContentChangeEvent], + line_index: &LineIndex, +) -> Result { + if changes.is_empty() { + return Ok(content.to_string()); + } + + // Check for full document replacement first + for change in changes { + if change.range.is_none() { + return Ok(change.text.clone()); + } + } + + // Sort changes by start position in reverse order (end to start) + let mut sorted_changes = changes.to_vec(); + sorted_changes.sort_by(|a, b| { + match (a.range, b.range) { + (Some(range_a), Some(range_b)) => { + // Primary sort: by line (reverse) + let line_cmp = range_b.start.line.cmp(&range_a.start.line); + if line_cmp != std::cmp::Ordering::Equal { + line_cmp + } else { + // Secondary sort: by character (reverse) + range_b.start.character.cmp(&range_a.start.character) + } + } + _ => std::cmp::Ordering::Equal, + } + }); + + let mut result = content.to_string(); + + for change in &sorted_changes { + if let Some(range) = change.range { + // Convert UTF-16 positions to UTF-8 offsets + let start_offset = line_index.offset_utf16(range.start, &result) + .ok_or_else(|| anyhow!("Invalid start position: {:?}", range.start))?; + let end_offset = line_index.offset_utf16(range.end, &result) + .ok_or_else(|| anyhow!("Invalid end position: {:?}", range.end))?; + + if start_offset as usize > result.len() || end_offset as usize > result.len() { + return Err(anyhow!("Offset out of bounds: start={}, end={}, len={}", + start_offset, end_offset, result.len())); + } + + // Apply the change + result.replace_range(start_offset as usize..end_offset as usize, &change.text); + } + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + use tower_lsp_server::lsp_types::Range; + + #[test] + fn test_apply_single_character_insertion() { + let content = "Hello world"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 6), Position::new(0, 6))), + range_length: None, + text: "beautiful ".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "Hello beautiful world"); + } + + #[test] + fn test_apply_single_character_deletion() { + let content = "Hello world"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 5), Position::new(0, 6))), + range_length: None, + text: "".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "Helloworld"); + } + + #[test] + fn test_apply_multiple_changes_in_reverse_order() { + let content = "line 1\nline 2\nline 3"; + let line_index = LineIndex::new(content); + + // Insert "new " at position (1, 0) and "another " at position (0, 0) + let changes = vec![ + TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), + range_length: None, + text: "another ".to_string(), + }, + TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(1, 0), Position::new(1, 0))), + range_length: None, + text: "new ".to_string(), + }, + ]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "another line 1\nnew line 2\nline 3"); + } + + #[test] + fn test_apply_multiline_replacement() { + let content = "line 1\nline 2\nline 3"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(2, 6))), + range_length: None, + text: "completely new content".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "completely new content"); + } + + #[test] + fn test_apply_full_document_replacement() { + let content = "old content"; + let line_index = LineIndex::new(content); + + let changes = vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "brand new content".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "brand new content"); + } + + #[test] + fn test_utf16_line_index_basic() { + let content = "hello world"; + let line_index = LineIndex::new(content); + + // ASCII characters should have 1:1 UTF-8:UTF-16 mapping + let pos = Position::new(0, 6); + let offset = line_index.offset_utf16(pos, content).unwrap(); + assert_eq!(offset, 6); + assert_eq!(&content[6..7], "w"); + } + + #[test] + fn test_utf16_line_index_with_emoji() { + let content = "hello 👋 world"; + let line_index = LineIndex::new(content); + + // 👋 is 2 UTF-16 code units but 4 UTF-8 bytes + let pos_after_emoji = Position::new(0, 8); // UTF-16 position after "hello 👋" + let offset = line_index.offset_utf16(pos_after_emoji, content).unwrap(); + + // Should point to the space before "world" + assert_eq!(offset, 10); // UTF-8 byte offset + assert_eq!(&content[10..11], " "); + } + + #[test] + fn test_utf16_line_index_multiline() { + let content = "first line\nsecond line"; + let line_index = LineIndex::new(content); + + let pos = Position::new(1, 7); // Position at 'l' in "line" on second line + let offset = line_index.offset_utf16(pos, content).unwrap(); + assert_eq!(offset, 18); // 11 (first line + \n) + 7 + assert_eq!(&content[18..19], "l"); + } + + #[test] + fn test_apply_changes_with_emoji() { + let content = "hello 👋 world"; + let line_index = LineIndex::new(content); + + // Insert text after the space following the emoji (UTF-16 position 9) + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 9), Position::new(0, 9))), + range_length: None, + text: "beautiful ".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "hello 👋 beautiful world"); + } + + #[test] + fn test_line_index_utf16_tracking() { + let content = "a👋b"; + let line_index = LineIndex::new(content); + + // Check UTF-16 line starts are tracked correctly + assert_eq!(line_index.line_starts_utf16, vec![0]); + assert_eq!(line_index.length_utf16, 4); // 'a' (1) + 👋 (2) + 'b' (1) = 4 UTF-16 units + assert_eq!(line_index.length, 6); // 'a' (1) + 👋 (4) + 'b' (1) = 6 UTF-8 bytes + } + + #[test] + fn test_edge_case_changes_at_boundaries() { + let content = "abc"; + let line_index = LineIndex::new(content); + + // Insert at beginning + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), + range_length: None, + text: "start".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "startabc"); + + // Insert at end + let line_index = LineIndex::new(content); + let changes = vec![TextDocumentContentChangeEvent { + range: Some(Range::new(Position::new(0, 3), Position::new(0, 3))), + range_length: None, + text: "end".to_string(), + }]; + + let result = apply_text_changes(content, &changes, &line_index).unwrap(); + assert_eq!(result, "abcend"); + } +} From 588b38c8c6d3412d6421a828a4cff594f341029f Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 04:31:21 -0500 Subject: [PATCH 04/30] wip --- crates/djls-server/src/db.rs | 22 ---------------- crates/djls-server/src/lib.rs | 1 - crates/djls-server/src/session.rs | 42 ------------------------------- 3 files changed, 65 deletions(-) delete mode 100644 crates/djls-server/src/db.rs diff --git a/crates/djls-server/src/db.rs b/crates/djls-server/src/db.rs deleted file mode 100644 index f6e4c33..0000000 --- a/crates/djls-server/src/db.rs +++ /dev/null @@ -1,22 +0,0 @@ -use salsa::Database; - -#[salsa::db] -#[derive(Clone, Default)] -pub struct ServerDatabase { - storage: salsa::Storage, -} - -impl ServerDatabase { - /// Create a new database from storage - pub fn new(storage: salsa::Storage) -> Self { - Self { storage } - } -} - -impl std::fmt::Debug for ServerDatabase { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ServerDatabase").finish_non_exhaustive() - } -} - -impl Database for ServerDatabase {} diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index 57c433a..1ebe618 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -1,5 +1,4 @@ mod client; -mod db; mod logging; mod queue; mod server; diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index d4e8aaf..a5832cb 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,10 +1,8 @@ use djls_conf::Settings; use djls_project::DjangoProject; -use salsa::StorageHandle; use tower_lsp_server::lsp_types::ClientCapabilities; use tower_lsp_server::lsp_types::InitializeParams; -use crate::db::ServerDatabase; use crate::workspace::Store; #[derive(Default)] @@ -15,36 +13,6 @@ pub struct Session { #[allow(dead_code)] client_capabilities: ClientCapabilities, - - /// A thread-safe Salsa database handle that can be shared between threads. - /// - /// This implements the insight from [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) - /// where we're using the `StorageHandle` to create a thread-safe handle that can be - /// shared between threads. When we need to use it, we clone the handle to get a new reference. - /// - /// This handle allows us to create database instances as needed. - /// Even though we're using a single-threaded runtime, we still need - /// this to be thread-safe because of LSP trait requirements. - /// - /// Usage: - /// ```rust,ignore - /// // Use the StorageHandle in Session - /// let db_handle = StorageHandle::new(None); - /// - /// // Clone it to pass to different threads - /// let db_handle_clone = db_handle.clone(); - /// - /// // Use it in an async context - /// async_fn(move || { - /// // Get a database from the handle - /// let storage = db_handle_clone.into_storage(); - /// let db = ServerDatabase::new(storage); - /// - /// // Use the database - /// db.some_query(args) - /// }); - /// ``` - db_handle: StorageHandle, } impl Session { @@ -67,7 +35,6 @@ impl Session { project, documents: Store::default(), settings, - db_handle: StorageHandle::new(None), } } @@ -94,13 +61,4 @@ impl Session { pub fn set_settings(&mut self, settings: Settings) { self.settings = settings; } - - /// Get a database instance directly from the session - /// - /// This creates a usable database from the handle, which can be used - /// to query and update data in the database. - pub fn db(&self) -> ServerDatabase { - let storage = self.db_handle.clone().into_storage(); - ServerDatabase::new(storage) - } } From b6bc1664ac9ccb895f7fd46dc06a6369e9c77aa6 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 05:54:56 -0500 Subject: [PATCH 05/30] wip --- Cargo.lock | 1 + crates/djls-server/src/workspace/store.rs | 79 +++++++++++- crates/djls-templates/src/ast.rs | 6 +- crates/djls-templates/src/lib.rs | 2 +- crates/djls-workspace/Cargo.toml | 2 + crates/djls-workspace/src/bridge.rs | 129 +++++++++++++++++++- crates/djls-workspace/src/db.rs | 142 ++++++++++++++++++++++ crates/djls-workspace/src/lib.rs | 2 +- 8 files changed, 354 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48cf438..3665fe4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,6 +509,7 @@ dependencies = [ "anyhow", "camino", "dashmap", + "djls-templates", "salsa", "url", ] diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 1a9efbb..6ecc537 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -5,24 +5,28 @@ use anyhow::anyhow; use anyhow::Result; use camino::Utf8PathBuf; use djls_project::TemplateTags; -use djls_workspace::{FileId, FileKind, TextSource, Vfs}; +use djls_workspace::{FileId, FileKind, FileStore, TextSource, Vfs}; use tower_lsp_server::lsp_types::CompletionItem; use tower_lsp_server::lsp_types::CompletionItemKind; use tower_lsp_server::lsp_types::CompletionResponse; +use tower_lsp_server::lsp_types::Diagnostic; +use tower_lsp_server::lsp_types::DiagnosticSeverity; use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use tower_lsp_server::lsp_types::Documentation; use tower_lsp_server::lsp_types::InsertTextFormat; use tower_lsp_server::lsp_types::MarkupContent; use tower_lsp_server::lsp_types::MarkupKind; use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; pub struct Store { vfs: Arc, + file_store: FileStore, file_ids: HashMap, line_indices: HashMap, versions: HashMap, @@ -33,6 +37,7 @@ impl Default for Store { fn default() -> Self { Self { vfs: Arc::new(Vfs::default()), + file_store: FileStore::new(), file_ids: HashMap::new(), line_indices: HashMap::new(), versions: HashMap::new(), @@ -64,6 +69,10 @@ impl Store { // Set overlay content in VFS self.vfs.set_overlay(file_id, Arc::from(content.as_str()))?; + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + self.file_store.apply_vfs_snapshot(&snapshot); + // Create TextDocument metadata let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); self.documents.insert(uri_str.clone(), document); @@ -111,6 +120,10 @@ impl Store { self.vfs .set_overlay(file_id, Arc::from(new_content.as_str()))?; + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + self.file_store.apply_vfs_snapshot(&snapshot); + // Update cached line index and version self.line_indices .insert(file_id, LineIndex::new(&new_content)); @@ -174,9 +187,18 @@ impl Store { return None; } + // Try to get cached AST from FileStore for better context analysis + // This demonstrates using the cached AST, though we still fall back to string parsing + let file_id = document.file_id(); + if let Some(_ast) = self.file_store.get_template_ast(file_id) { + // TODO: In a future enhancement, we could use the AST to provide + // more intelligent completions based on the current node context + // For now, we continue with the existing string-based approach + } + // Get template tag context from document let vfs_snapshot = self.vfs.snapshot(); - let line_index = self.get_line_index(document.file_id())?; + let line_index = self.get_line_index(file_id)?; let context = document.get_template_tag_context(&vfs_snapshot, line_index, position)?; let mut completions: Vec = tags @@ -214,6 +236,57 @@ impl Store { Some(CompletionResponse::Array(completions)) } } + + /// Get template parsing diagnostics for a file. + /// + /// This method uses the cached template errors from Salsa to generate LSP diagnostics. + /// The errors are only re-computed when the file content changes, providing efficient + /// incremental error reporting. + pub fn get_template_diagnostics(&self, uri: &str) -> Vec { + let Some(document) = self.get_document(uri) else { + return vec![]; + }; + + // Only process template files + if document.language_id != LanguageId::HtmlDjango { + return vec![]; + } + + let file_id = document.file_id(); + let Some(_line_index) = self.get_line_index(file_id) else { + return vec![]; + }; + + // Get cached template errors from FileStore + let errors = self.file_store.get_template_errors(file_id); + + // Convert template errors to LSP diagnostics + errors + .iter() + .map(|error| { + // For now, we'll place all errors at the start of the file + // In a future enhancement, we could use error spans for precise locations + let range = Range { + start: Position { + line: 0, + character: 0, + }, + end: Position { + line: 0, + character: 0, + }, + }; + + Diagnostic { + range, + severity: Some(DiagnosticSeverity::ERROR), + source: Some("djls-templates".to_string()), + message: error.clone(), + ..Default::default() + } + }) + .collect() + } } /// Apply text changes to content, handling multiple changes correctly diff --git a/crates/djls-templates/src/ast.rs b/crates/djls-templates/src/ast.rs index f355e70..1b62d4d 100644 --- a/crates/djls-templates/src/ast.rs +++ b/crates/djls-templates/src/ast.rs @@ -5,7 +5,7 @@ use crate::tokens::Token; use crate::tokens::TokenStream; use crate::tokens::TokenType; -#[derive(Clone, Debug, Default, Serialize)] +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] pub struct Ast { nodelist: Vec, line_offsets: LineOffsets, @@ -36,7 +36,7 @@ impl Ast { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub struct LineOffsets(pub Vec); impl LineOffsets { @@ -75,7 +75,7 @@ impl Default for LineOffsets { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] pub enum Node { Tag { name: String, diff --git a/crates/djls-templates/src/lib.rs b/crates/djls-templates/src/lib.rs index 4835056..7eab1f6 100644 --- a/crates/djls-templates/src/lib.rs +++ b/crates/djls-templates/src/lib.rs @@ -5,7 +5,7 @@ mod parser; mod tagspecs; mod tokens; -use ast::Ast; +pub use ast::Ast; pub use error::QuickFix; pub use error::TemplateError; use lexer::Lexer; diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 9b5ee6c..3d079cb 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -4,6 +4,8 @@ version = "0.0.0" edition = "2021" [dependencies] +djls-templates = { workspace = true } + anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 97f894e..45f92fc 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -9,7 +9,7 @@ use std::{collections::HashMap, sync::Arc}; use salsa::Setter; use super::{ - db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}, + db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}, vfs::{FileKind, VfsSnapshot}, FileId, }; @@ -99,6 +99,28 @@ impl FileStore { pub fn file_kind(&self, id: FileId) -> Option { self.files.get(&id).map(|sf| sf.kind(&self.db)) } + + /// Get the parsed template AST for a file by its [`FileId`]. + /// + /// This method leverages Salsa's incremental computation to cache parsed ASTs. + /// The AST is only re-parsed when the file's content changes in the VFS. + /// Returns `None` if the file is not tracked or is not a template file. + pub fn get_template_ast(&self, id: FileId) -> Option> { + let source_file = self.files.get(&id)?; + parse_template(&self.db, *source_file) + } + + /// Get template parsing errors for a file by its [`FileId`]. + /// + /// This method provides quick access to template errors without needing the full AST. + /// Useful for diagnostics and error reporting. Returns an empty slice for + /// non-template files or files not tracked in the store. + pub fn get_template_errors(&self, id: FileId) -> Arc<[String]> { + self.files + .get(&id) + .map(|sf| template_errors(&self.db, *sf)) + .unwrap_or_else(|| Arc::from(vec![])) + } } impl Default for FileStore { @@ -106,3 +128,108 @@ impl Default for FileStore { Self::new() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::vfs::{TextSource, Vfs}; + use camino::Utf8PathBuf; + + #[test] + fn test_filestore_template_ast_caching() { + let mut store = FileStore::new(); + let vfs = Vfs::default(); + + // Create a template file in VFS + let url = url::Url::parse("file:///test.html").unwrap(); + let path = Utf8PathBuf::from("/test.html"); + let content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); + let file_id = vfs.intern_file( + url.clone(), + path.clone(), + FileKind::Template, + TextSource::Overlay(content.clone()), + ); + vfs.set_overlay(file_id, content.clone()).unwrap(); + + // Apply VFS snapshot to FileStore + let snapshot = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot); + + // Get template AST - should parse and cache + let ast1 = store.get_template_ast(file_id); + assert!(ast1.is_some()); + + // Get again - should return cached + let ast2 = store.get_template_ast(file_id); + assert!(ast2.is_some()); + assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } + + #[test] + fn test_filestore_template_errors() { + let mut store = FileStore::new(); + let vfs = Vfs::default(); + + // Create a template with an unclosed tag + let url = url::Url::parse("file:///error.html").unwrap(); + let path = Utf8PathBuf::from("/error.html"); + let content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); // Missing closing + let file_id = vfs.intern_file( + url.clone(), + path.clone(), + FileKind::Template, + TextSource::Overlay(content.clone()), + ); + vfs.set_overlay(file_id, content).unwrap(); + + // Apply VFS snapshot + let snapshot = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot); + + // Get errors - should contain parsing errors + let errors = store.get_template_errors(file_id); + // The template has unclosed tags, so there should be errors + // We don't assert on specific error count as the parser may evolve + + // Verify errors are cached + let errors2 = store.get_template_errors(file_id); + assert!(Arc::ptr_eq(&errors, &errors2)); + } + + #[test] + fn test_filestore_invalidation_on_content_change() { + let mut store = FileStore::new(); + let vfs = Vfs::default(); + + // Create initial template + let url = url::Url::parse("file:///change.html").unwrap(); + let path = Utf8PathBuf::from("/change.html"); + let content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); + let file_id = vfs.intern_file( + url.clone(), + path.clone(), + FileKind::Template, + TextSource::Overlay(content1.clone()), + ); + vfs.set_overlay(file_id, content1).unwrap(); + + // Apply snapshot and get AST + let snapshot1 = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot1); + let ast1 = store.get_template_ast(file_id); + + // Change content + let content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); + vfs.set_overlay(file_id, content2).unwrap(); + + // Apply new snapshot + let snapshot2 = vfs.snapshot(); + store.apply_vfs_snapshot(&snapshot2); + + // Get AST again - should be different due to content change + let ast2 = store.get_template_ast(file_id); + assert!(ast1.is_some() && ast2.is_some()); + assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } +} diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 91aa1cc..bbdf0de 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -89,3 +89,145 @@ pub struct TemplateLoaderOrder { #[returns(ref)] pub roots: Arc<[String]>, } + +/// Container for a parsed Django template AST. +/// +/// [`TemplateAst`] wraps the parsed AST from djls-templates along with any parsing errors. +/// This struct is designed to be cached by Salsa and shared across multiple consumers +/// without re-parsing. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TemplateAst { + /// The parsed AST from djls-templates + pub ast: djls_templates::Ast, + /// Any errors encountered during parsing (stored as strings for simplicity) + pub errors: Vec, +} + +/// Parse a Django template file into an AST. +/// +/// This Salsa tracked function parses template files on-demand and caches the results. +/// The parse is only re-executed when the file's text content changes, enabling +/// efficient incremental template analysis. +/// +/// Returns `None` for non-template files. +#[salsa::tracked] +pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option> { + // Only parse template files + if file.kind(db) != FileKindMini::Template { + return None; + } + + let text = file.text(db); + + // Call the pure parsing function from djls-templates + match djls_templates::parse_template(&text) { + Ok((ast, errors)) => { + // Convert errors to strings + let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); + Some(Arc::new(TemplateAst { ast, errors: error_strings })) + }, + Err(err) => { + // Even on fatal errors, return an empty AST with the error + Some(Arc::new(TemplateAst { + ast: djls_templates::Ast::default(), + errors: vec![err.to_string()], + })) + } + } +} + +/// Get template parsing errors for a file. +/// +/// This Salsa tracked function extracts just the errors from the parsed template, +/// useful for diagnostics without needing the full AST. +/// +/// Returns an empty vector for non-template files. +#[salsa::tracked] +pub fn template_errors(db: &dyn salsa::Database, file: SourceFile) -> Arc<[String]> { + parse_template(db, file) + .map(|ast| Arc::from(ast.errors.clone())) + .unwrap_or_else(|| Arc::from(vec![])) +} + +#[cfg(test)] +mod tests { + use super::*; + use salsa::Setter; + + #[test] + fn test_template_parsing_caches_result() { + let db = Database::default(); + + // Create a template file + let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); + let file = SourceFile::new(&db, FileKindMini::Template, template_content.clone()); + + // First parse - should execute the parsing + let ast1 = parse_template(&db, file); + assert!(ast1.is_some()); + + // Second parse - should return cached result (same Arc) + let ast2 = parse_template(&db, file); + assert!(ast2.is_some()); + + // Verify they're the same Arc (cached) + assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } + + #[test] + fn test_template_parsing_invalidates_on_change() { + let mut db = Database::default(); + + // Create a template file + let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); + let file = SourceFile::new(&db, FileKindMini::Template, template_content1); + + // First parse + let ast1 = parse_template(&db, file); + assert!(ast1.is_some()); + + // Change the content + let template_content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); + file.set_text(&mut db).to(template_content2); + + // Parse again - should re-execute due to changed content + let ast2 = parse_template(&db, file); + assert!(ast2.is_some()); + + // Verify they're different Arcs (re-parsed) + assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); + } + + #[test] + fn test_non_template_files_return_none() { + let db = Database::default(); + + // Create a Python file + let python_content: Arc = Arc::from("def hello():\n print('Hello')"); + let file = SourceFile::new(&db, FileKindMini::Python, python_content); + + // Should return None for non-template files + let ast = parse_template(&db, file); + assert!(ast.is_none()); + + // Errors should be empty for non-template files + let errors = template_errors(&db, file); + assert!(errors.is_empty()); + } + + #[test] + fn test_template_errors_tracked_separately() { + let db = Database::default(); + + // Create a template with an error (unclosed tag) + let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); + let file = SourceFile::new(&db, FileKindMini::Template, template_content); + + // Get errors + let errors1 = template_errors(&db, file); + let errors2 = template_errors(&db, file); + + // Should be cached (same Arc) + assert!(Arc::ptr_eq(&errors1, &errors2)); + } +} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index aea4a53..7eb862a 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -4,7 +4,7 @@ mod vfs; // Re-export public API pub use bridge::FileStore; -pub use db::{Database, FileKindMini, SourceFile, TemplateLoaderOrder}; +pub use db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; /// Stable, compact identifier for files across the subsystem. From fb768a86d5f126bd3198fa92eb37d65254677e0e Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 10:14:43 -0500 Subject: [PATCH 06/30] wip --- Cargo.lock | 105 +++++++ Cargo.toml | 1 + crates/djls-server/src/workspace/store.rs | 18 +- crates/djls-workspace/Cargo.toml | 3 + crates/djls-workspace/src/lib.rs | 2 + crates/djls-workspace/src/vfs.rs | 101 ++++++- crates/djls-workspace/src/watcher.rs | 330 ++++++++++++++++++++++ 7 files changed, 551 insertions(+), 9 deletions(-) create mode 100644 crates/djls-workspace/src/watcher.rs diff --git a/Cargo.lock b/Cargo.lock index 3665fe4..73c6aae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -510,7 +510,10 @@ dependencies = [ "camino", "dashmap", "djls-templates", + "notify", "salsa", + "tempfile", + "tokio", "url", ] @@ -606,6 +609,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futures" version = "0.3.31" @@ -883,6 +895,26 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.9.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "insta" version = "1.43.1" @@ -938,6 +970,26 @@ dependencies = [ "serde", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1041,10 +1093,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", + "log", "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.59.0", ] +[[package]] +name = "notify" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" +dependencies = [ + "bitflags 2.9.2", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.60.2", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1493,6 +1570,15 @@ dependencies = [ "synstructure", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2074,6 +2160,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2116,6 +2212,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 8a88a0f..6d18e3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ clap = { version = "4.5", features = ["derive"] } config = { version ="0.15", features = ["toml"] } dashmap = "6.1" directories = "6.0" +notify = "8.2" percent-encoding = "2.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 6ecc537..5a38d37 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use anyhow::anyhow; use anyhow::Result; @@ -26,7 +26,7 @@ use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; pub struct Store { vfs: Arc, - file_store: FileStore, + file_store: Arc>, file_ids: HashMap, line_indices: HashMap, versions: HashMap, @@ -37,7 +37,7 @@ impl Default for Store { fn default() -> Self { Self { vfs: Arc::new(Vfs::default()), - file_store: FileStore::new(), + file_store: Arc::new(Mutex::new(FileStore::new())), file_ids: HashMap::new(), line_indices: HashMap::new(), versions: HashMap::new(), @@ -71,7 +71,8 @@ impl Store { // Sync VFS snapshot to FileStore for Salsa tracking let snapshot = self.vfs.snapshot(); - self.file_store.apply_vfs_snapshot(&snapshot); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); // Create TextDocument metadata let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); @@ -122,7 +123,8 @@ impl Store { // Sync VFS snapshot to FileStore for Salsa tracking let snapshot = self.vfs.snapshot(); - self.file_store.apply_vfs_snapshot(&snapshot); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); // Update cached line index and version self.line_indices @@ -190,7 +192,8 @@ impl Store { // Try to get cached AST from FileStore for better context analysis // This demonstrates using the cached AST, though we still fall back to string parsing let file_id = document.file_id(); - if let Some(_ast) = self.file_store.get_template_ast(file_id) { + let file_store = self.file_store.lock().unwrap(); + if let Some(_ast) = file_store.get_template_ast(file_id) { // TODO: In a future enhancement, we could use the AST to provide // more intelligent completions based on the current node context // For now, we continue with the existing string-based approach @@ -258,7 +261,8 @@ impl Store { }; // Get cached template errors from FileStore - let errors = self.file_store.get_template_errors(file_id); + let file_store = self.file_store.lock().unwrap(); + let errors = file_store.get_template_errors(file_id); // Convert template errors to LSP diagnostics errors diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 3d079cb..34fa01e 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -9,10 +9,13 @@ djls-templates = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } +notify = { workspace = true } salsa = { workspace = true } +tokio = { workspace = true } url = { workspace = true } [dev-dependencies] +tempfile = { workspace = true } [lints] workspace = true diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 7eb862a..83fc7d0 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,11 +1,13 @@ mod bridge; mod db; mod vfs; +mod watcher; // Re-export public API pub use bridge::FileStore; pub use db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; +pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; /// Stable, compact identifier for files across the subsystem. /// diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs index 9ebe6a2..be3ad18 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs.rs @@ -18,7 +18,7 @@ use std::{ }; use url::Url; -use super::FileId; +use super::{FileId, watcher::{VfsWatcher, WatchConfig, WatchEvent}}; /// Monotonic counter representing global VFS state. /// @@ -113,6 +113,10 @@ pub struct Vfs { files: DashMap, /// Global revision counter, incremented on content changes head: AtomicU64, + /// Optional file system watcher for external change detection + watcher: std::sync::Mutex>, + /// Map from filesystem path to FileId for watcher events + by_path: DashMap, } impl Vfs { @@ -134,11 +138,12 @@ impl Vfs { let id = FileId(self.next_file_id.fetch_add(1, Ordering::SeqCst)); let meta = FileMeta { uri: uri.clone(), - path, + path: path.clone(), kind, }; let hash = content_hash(&text); self.by_uri.insert(uri, id); + self.by_path.insert(path, id); self.files.insert(id, FileRecord { meta, text, hash }); id } @@ -188,6 +193,96 @@ impl Vfs { .collect(), } } + + /// Enable file system watching with the given configuration. + /// + /// This starts monitoring the specified root directories for external changes. + /// Returns an error if file watching is disabled in the config or fails to start. + pub fn enable_file_watching(&self, config: WatchConfig) -> Result<()> { + let watcher = VfsWatcher::new(config)?; + *self.watcher.lock().unwrap() = Some(watcher); + Ok(()) + } + + /// Process pending file system events from the watcher. + /// + /// This should be called periodically to sync external file changes into the VFS. + /// Returns the number of files that were updated. + pub fn process_file_events(&self) -> usize { + // Get events from the watcher + let events = { + let guard = self.watcher.lock().unwrap(); + if let Some(watcher) = guard.as_ref() { + watcher.try_recv_events() + } else { + return 0; + } + }; + + let mut updated_count = 0; + + for event in events { + match event { + WatchEvent::Modified(path) | WatchEvent::Created(path) => { + if let Err(e) = self.load_from_disk(&path) { + eprintln!("Failed to load file from disk: {}: {}", path, e); + } else { + updated_count += 1; + } + } + WatchEvent::Deleted(path) => { + // For now, we don't remove deleted files from VFS + // This maintains stable FileIds for consumers + eprintln!("File deleted (keeping in VFS): {}", path); + } + WatchEvent::Renamed { from, to } => { + // Handle rename by updating the path mapping + if let Some(file_id) = self.by_path.remove(&from).map(|(_, id)| id) { + self.by_path.insert(to.clone(), file_id); + if let Err(e) = self.load_from_disk(&to) { + eprintln!("Failed to load renamed file: {}: {}", to, e); + } else { + updated_count += 1; + } + } + } + } + } + updated_count + } + + /// Load a file's content from disk and update the VFS. + /// + /// This method reads the file from the filesystem and updates the VFS entry + /// if the content has changed. It's used by the file watcher to sync external changes. + fn load_from_disk(&self, path: &Utf8PathBuf) -> Result<()> { + use std::fs; + + // Check if we have this file tracked + if let Some(file_id) = self.by_path.get(path).map(|entry| *entry.value()) { + // Read content from disk + let content = fs::read_to_string(path.as_std_path()) + .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; + + let new_text = TextSource::Disk(Arc::from(content.as_str())); + let new_hash = content_hash(&new_text); + + // Update the file if content changed + if let Some(mut record) = self.files.get_mut(&file_id) { + if record.hash != new_hash { + record.text = new_text; + record.hash = new_hash; + self.head.fetch_add(1, Ordering::SeqCst); + } + } + } + Ok(()) + } + + /// Check if file watching is currently enabled. + pub fn is_file_watching_enabled(&self) -> bool { + self.watcher.lock().unwrap().is_some() + } } impl Default for Vfs { @@ -197,6 +292,8 @@ impl Default for Vfs { by_uri: DashMap::new(), files: DashMap::new(), head: AtomicU64::new(0), + watcher: std::sync::Mutex::new(None), + by_path: DashMap::new(), } } } diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/watcher.rs new file mode 100644 index 0000000..cd6d958 --- /dev/null +++ b/crates/djls-workspace/src/watcher.rs @@ -0,0 +1,330 @@ +//! File system watching for VFS synchronization. +//! +//! This module provides file system watching capabilities to detect external changes +//! and synchronize them with the VFS. It uses cross-platform file watching with +//! debouncing to handle rapid changes efficiently. + +use anyhow::{anyhow, Result}; +use camino::Utf8PathBuf; +use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; +use std::{ + collections::HashMap, + sync::mpsc, + thread, + time::{Duration, Instant}, +}; + +/// Event types that can occur in the file system. +/// +/// [`WatchEvent`] represents the different types of file system changes that +/// the watcher can detect and process. +#[derive(Clone, Debug, PartialEq)] +pub enum WatchEvent { + /// A file was modified (content changed) + Modified(Utf8PathBuf), + /// A new file was created + Created(Utf8PathBuf), + /// A file was deleted + Deleted(Utf8PathBuf), + /// A file was renamed from one path to another + Renamed { + from: Utf8PathBuf, + to: Utf8PathBuf, + }, +} + +/// Configuration for the file watcher. +/// +/// [`WatchConfig`] controls how the file watcher operates, including what +/// directories to watch and how to filter events. +#[derive(Clone, Debug)] +pub struct WatchConfig { + /// Whether file watching is enabled + pub enabled: bool, + /// Root directories to watch recursively + pub roots: Vec, + /// Debounce time in milliseconds (collect events for this duration before processing) + pub debounce_ms: u64, + /// File patterns to include (e.g., ["*.py", "*.html"]) + pub include_patterns: Vec, + /// File patterns to exclude (e.g., ["__pycache__", ".git", "*.pyc"]) + pub exclude_patterns: Vec, +} + +impl Default for WatchConfig { + fn default() -> Self { + Self { + enabled: true, + roots: Vec::new(), + debounce_ms: 250, + include_patterns: vec!["*.py".to_string(), "*.html".to_string()], + exclude_patterns: vec![ + "__pycache__".to_string(), + ".git".to_string(), + ".pyc".to_string(), + "node_modules".to_string(), + ".venv".to_string(), + "venv".to_string(), + ], + } + } +} + +/// File system watcher for VFS synchronization. +/// +/// [`VfsWatcher`] monitors the file system for changes and provides a channel +/// for consuming batched events. It handles debouncing and filtering internally. +pub struct VfsWatcher { + /// The underlying file system watcher + _watcher: RecommendedWatcher, + /// Receiver for processed watch events + rx: mpsc::Receiver>, + /// Configuration for the watcher + config: WatchConfig, + /// Handle to the background processing thread + _handle: thread::JoinHandle<()>, +} + +impl VfsWatcher { + /// Create a new file watcher with the given configuration. + /// + /// This starts watching the specified root directories and begins processing + /// events in a background thread. + pub fn new(config: WatchConfig) -> Result { + if !config.enabled { + return Err(anyhow!("File watching is disabled")); + } + + let (event_tx, event_rx) = mpsc::channel(); + let (watch_tx, watch_rx) = mpsc::channel(); + + // Create the file system watcher + let mut watcher = RecommendedWatcher::new( + move |res: notify::Result| { + if let Ok(event) = res { + let _ = event_tx.send(event); + } + }, + Config::default(), + )?; + + // Watch all root directories + for root in &config.roots { + let std_path = root.as_std_path(); + if std_path.exists() { + watcher.watch(std_path, RecursiveMode::Recursive)?; + } + } + + // Spawn background thread for event processing + let config_clone = config.clone(); + let handle = thread::spawn(move || { + Self::process_events(event_rx, watch_tx, config_clone); + }); + + Ok(Self { + _watcher: watcher, + rx: watch_rx, + config, + _handle: handle, + }) + } + + /// Get the next batch of processed watch events. + /// + /// This is a non-blocking operation that returns immediately. If no events + /// are available, it returns an empty vector. + pub fn try_recv_events(&self) -> Vec { + match self.rx.try_recv() { + Ok(events) => events, + Err(_) => Vec::new(), + } + } + + + + /// Background thread function for processing raw file system events. + /// + /// This function handles debouncing, filtering, and batching of events before + /// sending them to the main thread for VFS synchronization. + fn process_events( + event_rx: mpsc::Receiver, + watch_tx: mpsc::Sender>, + config: WatchConfig, + ) { + let mut pending_events: HashMap = HashMap::new(); + let mut last_batch_time = Instant::now(); + let debounce_duration = Duration::from_millis(config.debounce_ms); + + loop { + // Try to receive events with a timeout for batching + match event_rx.recv_timeout(Duration::from_millis(50)) { + Ok(event) => { + // Process the raw notify event into our WatchEvent format + if let Some(watch_events) = Self::convert_notify_event(event, &config) { + for watch_event in watch_events { + if let Some(path) = Self::get_event_path(&watch_event) { + // Only keep the latest event for each path + pending_events.insert(path.clone(), watch_event); + } + } + } + } + Err(mpsc::RecvTimeoutError::Timeout) => { + // Timeout - check if we should flush pending events + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + // Channel disconnected, exit the thread + break; + } + } + + // Check if we should flush pending events + if !pending_events.is_empty() + && last_batch_time.elapsed() >= debounce_duration + { + let events: Vec = pending_events.values().cloned().collect(); + if let Err(_) = watch_tx.send(events) { + // Main thread disconnected, exit + break; + } + pending_events.clear(); + last_batch_time = Instant::now(); + } + } + } + + /// Convert a notify Event into our WatchEvent format. + fn convert_notify_event(event: Event, config: &WatchConfig) -> Option> { + let mut watch_events = Vec::new(); + + for path in event.paths { + if let Ok(utf8_path) = Utf8PathBuf::try_from(path) { + if Self::should_include_path_static(&utf8_path, config) { + match event.kind { + EventKind::Create(_) => watch_events.push(WatchEvent::Created(utf8_path)), + EventKind::Modify(_) => watch_events.push(WatchEvent::Modified(utf8_path)), + EventKind::Remove(_) => watch_events.push(WatchEvent::Deleted(utf8_path)), + _ => {} // Ignore other event types for now + } + } + } + } + + if watch_events.is_empty() { + None + } else { + Some(watch_events) + } + } + + /// Static version of should_include_path for use in convert_notify_event. + fn should_include_path_static(path: &Utf8PathBuf, config: &WatchConfig) -> bool { + let path_str = path.as_str(); + + // Check exclude patterns first + for pattern in &config.exclude_patterns { + if path_str.contains(pattern) { + return false; + } + } + + // If no include patterns, include everything (that's not excluded) + if config.include_patterns.is_empty() { + return true; + } + + // Check include patterns + for pattern in &config.include_patterns { + if pattern.starts_with("*.") { + let extension = &pattern[2..]; + if path_str.ends_with(extension) { + return true; + } + } else if path_str.contains(pattern) { + return true; + } + } + + false + } + + /// Extract the path from a WatchEvent. + fn get_event_path(event: &WatchEvent) -> Option<&Utf8PathBuf> { + match event { + WatchEvent::Modified(path) => Some(path), + WatchEvent::Created(path) => Some(path), + WatchEvent::Deleted(path) => Some(path), + WatchEvent::Renamed { to, .. } => Some(to), + } + } +} + +impl Drop for VfsWatcher { + fn drop(&mut self) { + // The background thread will exit when the event channel is dropped + } +} + +#[cfg(test)] +mod tests { + use super::*; + + + #[test] + fn test_watch_config_default() { + let config = WatchConfig::default(); + assert!(config.enabled); + assert_eq!(config.debounce_ms, 250); + assert!(config.include_patterns.contains(&"*.py".to_string())); + assert!(config.exclude_patterns.contains(&".git".to_string())); + } + + #[test] + fn test_should_include_path() { + let config = WatchConfig::default(); + + // Should include Python files + assert!(VfsWatcher::should_include_path_static( + &Utf8PathBuf::from("test.py"), + &config + )); + + // Should include HTML files + assert!(VfsWatcher::should_include_path_static( + &Utf8PathBuf::from("template.html"), + &config + )); + + // Should exclude .git files + assert!(!VfsWatcher::should_include_path_static( + &Utf8PathBuf::from(".git/config"), + &config + )); + + // Should exclude __pycache__ files + assert!(!VfsWatcher::should_include_path_static( + &Utf8PathBuf::from("__pycache__/test.pyc"), + &config + )); + } + + #[test] + fn test_watch_event_types() { + let path1 = Utf8PathBuf::from("test.py"); + let path2 = Utf8PathBuf::from("new.py"); + + let modified = WatchEvent::Modified(path1.clone()); + let created = WatchEvent::Created(path1.clone()); + let deleted = WatchEvent::Deleted(path1.clone()); + let renamed = WatchEvent::Renamed { + from: path1, + to: path2, + }; + + // Test that events can be created and compared + assert_ne!(modified, created); + assert_ne!(created, deleted); + assert_ne!(deleted, renamed); + } +} \ No newline at end of file From 20163b50f8eb0cb1b896e8496884eb756acd27db Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 10:34:57 -0500 Subject: [PATCH 07/30] wip --- crates/djls-workspace/src/bridge.rs | 35 +++++++++-------- crates/djls-workspace/src/db.rs | 48 ++++++++++++----------- crates/djls-workspace/src/lib.rs | 6 ++- crates/djls-workspace/src/vfs.rs | 41 +++++++++++--------- crates/djls-workspace/src/watcher.rs | 58 ++++++++++++---------------- 5 files changed, 94 insertions(+), 94 deletions(-) diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 45f92fc..78669ad 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -9,7 +9,10 @@ use std::{collections::HashMap, sync::Arc}; use salsa::Setter; use super::{ - db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}, + db::{ + parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, + TemplateLoaderOrder, + }, vfs::{FileKind, VfsSnapshot}, FileId, }; @@ -31,6 +34,7 @@ pub struct FileStore { impl FileStore { /// Construct an empty store and DB. + #[must_use] pub fn new() -> Self { Self { db: Database::default(), @@ -118,8 +122,7 @@ impl FileStore { pub fn get_template_errors(&self, id: FileId) -> Arc<[String]> { self.files .get(&id) - .map(|sf| template_errors(&self.db, *sf)) - .unwrap_or_else(|| Arc::from(vec![])) + .map_or_else(|| Arc::from(vec![]), |sf| template_errors(&self.db, *sf)) } } @@ -139,7 +142,7 @@ mod tests { fn test_filestore_template_ast_caching() { let mut store = FileStore::new(); let vfs = Vfs::default(); - + // Create a template file in VFS let url = url::Url::parse("file:///test.html").unwrap(); let path = Utf8PathBuf::from("/test.html"); @@ -151,15 +154,15 @@ mod tests { TextSource::Overlay(content.clone()), ); vfs.set_overlay(file_id, content.clone()).unwrap(); - + // Apply VFS snapshot to FileStore let snapshot = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot); - + // Get template AST - should parse and cache let ast1 = store.get_template_ast(file_id); assert!(ast1.is_some()); - + // Get again - should return cached let ast2 = store.get_template_ast(file_id); assert!(ast2.is_some()); @@ -170,7 +173,7 @@ mod tests { fn test_filestore_template_errors() { let mut store = FileStore::new(); let vfs = Vfs::default(); - + // Create a template with an unclosed tag let url = url::Url::parse("file:///error.html").unwrap(); let path = Utf8PathBuf::from("/error.html"); @@ -182,16 +185,16 @@ mod tests { TextSource::Overlay(content.clone()), ); vfs.set_overlay(file_id, content).unwrap(); - + // Apply VFS snapshot let snapshot = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot); - + // Get errors - should contain parsing errors let errors = store.get_template_errors(file_id); // The template has unclosed tags, so there should be errors // We don't assert on specific error count as the parser may evolve - + // Verify errors are cached let errors2 = store.get_template_errors(file_id); assert!(Arc::ptr_eq(&errors, &errors2)); @@ -201,7 +204,7 @@ mod tests { fn test_filestore_invalidation_on_content_change() { let mut store = FileStore::new(); let vfs = Vfs::default(); - + // Create initial template let url = url::Url::parse("file:///change.html").unwrap(); let path = Utf8PathBuf::from("/change.html"); @@ -213,20 +216,20 @@ mod tests { TextSource::Overlay(content1.clone()), ); vfs.set_overlay(file_id, content1).unwrap(); - + // Apply snapshot and get AST let snapshot1 = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot1); let ast1 = store.get_template_ast(file_id); - + // Change content let content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); vfs.set_overlay(file_id, content2).unwrap(); - + // Apply new snapshot let snapshot2 = vfs.snapshot(); store.apply_vfs_snapshot(&snapshot2); - + // Get AST again - should be different due to content change let ast2 = store.get_template_ast(file_id); assert!(ast1.is_some() && ast2.is_some()); diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index bbdf0de..a38757d 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -118,14 +118,17 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option { // Convert errors to strings let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); - Some(Arc::new(TemplateAst { ast, errors: error_strings })) - }, + Some(Arc::new(TemplateAst { + ast, + errors: error_strings, + })) + } Err(err) => { // Even on fatal errors, return an empty AST with the error Some(Arc::new(TemplateAst { @@ -144,9 +147,7 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option Arc<[String]> { - parse_template(db, file) - .map(|ast| Arc::from(ast.errors.clone())) - .unwrap_or_else(|| Arc::from(vec![])) + parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } #[cfg(test)] @@ -157,19 +158,19 @@ mod tests { #[test] fn test_template_parsing_caches_result() { let db = Database::default(); - + // Create a template file let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); let file = SourceFile::new(&db, FileKindMini::Template, template_content.clone()); - + // First parse - should execute the parsing let ast1 = parse_template(&db, file); assert!(ast1.is_some()); - + // Second parse - should return cached result (same Arc) let ast2 = parse_template(&db, file); assert!(ast2.is_some()); - + // Verify they're the same Arc (cached) assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); } @@ -177,23 +178,24 @@ mod tests { #[test] fn test_template_parsing_invalidates_on_change() { let mut db = Database::default(); - + // Create a template file let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); let file = SourceFile::new(&db, FileKindMini::Template, template_content1); - + // First parse let ast1 = parse_template(&db, file); assert!(ast1.is_some()); - + // Change the content - let template_content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); + let template_content2: Arc = + Arc::from("{% for item in items %}{{ item }}{% endfor %}"); file.set_text(&mut db).to(template_content2); - + // Parse again - should re-execute due to changed content let ast2 = parse_template(&db, file); assert!(ast2.is_some()); - + // Verify they're different Arcs (re-parsed) assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); } @@ -201,15 +203,15 @@ mod tests { #[test] fn test_non_template_files_return_none() { let db = Database::default(); - + // Create a Python file let python_content: Arc = Arc::from("def hello():\n print('Hello')"); let file = SourceFile::new(&db, FileKindMini::Python, python_content); - + // Should return None for non-template files let ast = parse_template(&db, file); assert!(ast.is_none()); - + // Errors should be empty for non-template files let errors = template_errors(&db, file); assert!(errors.is_empty()); @@ -218,15 +220,15 @@ mod tests { #[test] fn test_template_errors_tracked_separately() { let db = Database::default(); - + // Create a template with an error (unclosed tag) let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); let file = SourceFile::new(&db, FileKindMini::Template, template_content); - + // Get errors let errors1 = template_errors(&db, file); let errors2 = template_errors(&db, file); - + // Should be cached (same Arc) assert!(Arc::ptr_eq(&errors1, &errors2)); } diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 83fc7d0..30c69a7 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -3,9 +3,11 @@ mod db; mod vfs; mod watcher; -// Re-export public API pub use bridge::FileStore; -pub use db::{parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, TemplateLoaderOrder}; +pub use db::{ + parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, + TemplateLoaderOrder, +}; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs.rs index be3ad18..08eb182 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs.rs @@ -8,6 +8,7 @@ use anyhow::{anyhow, Result}; use camino::Utf8PathBuf; use dashmap::DashMap; use std::collections::hash_map::DefaultHasher; +use std::fs; use std::hash::{Hash, Hasher}; use std::{ collections::HashMap, @@ -18,7 +19,10 @@ use std::{ }; use url::Url; -use super::{FileId, watcher::{VfsWatcher, WatchConfig, WatchEvent}}; +use super::{ + watcher::{VfsWatcher, WatchConfig, WatchEvent}, + FileId, +}; /// Monotonic counter representing global VFS state. /// @@ -115,7 +119,7 @@ pub struct Vfs { head: AtomicU64, /// Optional file system watcher for external change detection watcher: std::sync::Mutex>, - /// Map from filesystem path to FileId for watcher events + /// Map from filesystem path to [`FileId`] for watcher events by_path: DashMap, } @@ -155,10 +159,6 @@ impl Vfs { /// (detected via hash comparison). /// /// Returns a tuple of (new global revision, whether content changed). - /// - /// # Errors - /// - /// Returns an error if the provided `FileId` does not exist in the VFS. pub fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { let mut rec = self .files @@ -200,7 +200,10 @@ impl Vfs { /// Returns an error if file watching is disabled in the config or fails to start. pub fn enable_file_watching(&self, config: WatchConfig) -> Result<()> { let watcher = VfsWatcher::new(config)?; - *self.watcher.lock().unwrap() = Some(watcher); + *self + .watcher + .lock() + .map_err(|e| anyhow!("Failed to lock watcher mutex: {}", e))? = Some(watcher); Ok(()) } @@ -211,36 +214,38 @@ impl Vfs { pub fn process_file_events(&self) -> usize { // Get events from the watcher let events = { - let guard = self.watcher.lock().unwrap(); + let Ok(guard) = self.watcher.lock() else { + return 0; // Return 0 if mutex is poisoned + }; if let Some(watcher) = guard.as_ref() { watcher.try_recv_events() } else { return 0; } }; - + let mut updated_count = 0; - + for event in events { match event { WatchEvent::Modified(path) | WatchEvent::Created(path) => { if let Err(e) = self.load_from_disk(&path) { - eprintln!("Failed to load file from disk: {}: {}", path, e); + eprintln!("Failed to load file from disk: {path}: {e}"); } else { updated_count += 1; } } WatchEvent::Deleted(path) => { // For now, we don't remove deleted files from VFS - // This maintains stable FileIds for consumers - eprintln!("File deleted (keeping in VFS): {}", path); + // This maintains stable `FileId`s for consumers + eprintln!("File deleted (keeping in VFS): {path}"); } WatchEvent::Renamed { from, to } => { // Handle rename by updating the path mapping if let Some(file_id) = self.by_path.remove(&from).map(|(_, id)| id) { self.by_path.insert(to.clone(), file_id); if let Err(e) = self.load_from_disk(&to) { - eprintln!("Failed to load renamed file: {}: {}", to, e); + eprintln!("Failed to load renamed file: {to}: {e}"); } else { updated_count += 1; } @@ -256,17 +261,15 @@ impl Vfs { /// This method reads the file from the filesystem and updates the VFS entry /// if the content has changed. It's used by the file watcher to sync external changes. fn load_from_disk(&self, path: &Utf8PathBuf) -> Result<()> { - use std::fs; - // Check if we have this file tracked if let Some(file_id) = self.by_path.get(path).map(|entry| *entry.value()) { // Read content from disk let content = fs::read_to_string(path.as_std_path()) .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; - + let new_text = TextSource::Disk(Arc::from(content.as_str())); let new_hash = content_hash(&new_text); - + // Update the file if content changed if let Some(mut record) = self.files.get_mut(&file_id) { if record.hash != new_hash { @@ -281,7 +284,7 @@ impl Vfs { /// Check if file watching is currently enabled. pub fn is_file_watching_enabled(&self) -> bool { - self.watcher.lock().unwrap().is_some() + self.watcher.lock().map(|g| g.is_some()).unwrap_or(false) // Return false if mutex is poisoned } } diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/watcher.rs index cd6d958..57c798c 100644 --- a/crates/djls-workspace/src/watcher.rs +++ b/crates/djls-workspace/src/watcher.rs @@ -27,10 +27,7 @@ pub enum WatchEvent { /// A file was deleted Deleted(Utf8PathBuf), /// A file was renamed from one path to another - Renamed { - from: Utf8PathBuf, - to: Utf8PathBuf, - }, + Renamed { from: Utf8PathBuf, to: Utf8PathBuf }, } /// Configuration for the file watcher. @@ -51,6 +48,7 @@ pub struct WatchConfig { pub exclude_patterns: Vec, } +// TODO: Allow for user config instead of hardcoding defaults impl Default for WatchConfig { fn default() -> Self { Self { @@ -119,7 +117,7 @@ impl VfsWatcher { // Spawn background thread for event processing let config_clone = config.clone(); let handle = thread::spawn(move || { - Self::process_events(event_rx, watch_tx, config_clone); + Self::process_events(&event_rx, &watch_tx, &config_clone); }); Ok(Self { @@ -134,23 +132,19 @@ impl VfsWatcher { /// /// This is a non-blocking operation that returns immediately. If no events /// are available, it returns an empty vector. + #[must_use] pub fn try_recv_events(&self) -> Vec { - match self.rx.try_recv() { - Ok(events) => events, - Err(_) => Vec::new(), - } + self.rx.try_recv().unwrap_or_default() } - - /// Background thread function for processing raw file system events. /// /// This function handles debouncing, filtering, and batching of events before /// sending them to the main thread for VFS synchronization. fn process_events( - event_rx: mpsc::Receiver, - watch_tx: mpsc::Sender>, - config: WatchConfig, + event_rx: &mpsc::Receiver, + watch_tx: &mpsc::Sender>, + config: &WatchConfig, ) { let mut pending_events: HashMap = HashMap::new(); let mut last_batch_time = Instant::now(); @@ -161,12 +155,11 @@ impl VfsWatcher { match event_rx.recv_timeout(Duration::from_millis(50)) { Ok(event) => { // Process the raw notify event into our WatchEvent format - if let Some(watch_events) = Self::convert_notify_event(event, &config) { + if let Some(watch_events) = Self::convert_notify_event(event, config) { for watch_event in watch_events { - if let Some(path) = Self::get_event_path(&watch_event) { - // Only keep the latest event for each path - pending_events.insert(path.clone(), watch_event); - } + let path = Self::get_event_path(&watch_event); + // Only keep the latest event for each path + pending_events.insert(path.clone(), watch_event); } } } @@ -180,11 +173,9 @@ impl VfsWatcher { } // Check if we should flush pending events - if !pending_events.is_empty() - && last_batch_time.elapsed() >= debounce_duration - { + if !pending_events.is_empty() && last_batch_time.elapsed() >= debounce_duration { let events: Vec = pending_events.values().cloned().collect(); - if let Err(_) = watch_tx.send(events) { + if watch_tx.send(events).is_err() { // Main thread disconnected, exit break; } @@ -194,7 +185,7 @@ impl VfsWatcher { } } - /// Convert a notify Event into our WatchEvent format. + /// Convert a [`notify::Event`] into our [`WatchEvent`] format. fn convert_notify_event(event: Event, config: &WatchConfig) -> Option> { let mut watch_events = Vec::new(); @@ -236,8 +227,7 @@ impl VfsWatcher { // Check include patterns for pattern in &config.include_patterns { - if pattern.starts_with("*.") { - let extension = &pattern[2..]; + if let Some(extension) = pattern.strip_prefix("*.") { if path_str.ends_with(extension) { return true; } @@ -249,13 +239,13 @@ impl VfsWatcher { false } - /// Extract the path from a WatchEvent. - fn get_event_path(event: &WatchEvent) -> Option<&Utf8PathBuf> { + /// Extract the path from a [`WatchEvent`]. + fn get_event_path(event: &WatchEvent) -> &Utf8PathBuf { match event { - WatchEvent::Modified(path) => Some(path), - WatchEvent::Created(path) => Some(path), - WatchEvent::Deleted(path) => Some(path), - WatchEvent::Renamed { to, .. } => Some(to), + WatchEvent::Modified(path) | WatchEvent::Created(path) | WatchEvent::Deleted(path) => { + path + } + WatchEvent::Renamed { to, .. } => to, } } } @@ -270,7 +260,6 @@ impl Drop for VfsWatcher { mod tests { use super::*; - #[test] fn test_watch_config_default() { let config = WatchConfig::default(); @@ -327,4 +316,5 @@ mod tests { assert_ne!(created, deleted); assert_ne!(deleted, renamed); } -} \ No newline at end of file +} + From 541200cbb13076e0d72eaf41b67a0802848cd5c3 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 10:52:37 -0500 Subject: [PATCH 08/30] wip --- crates/djls-server/src/workspace/document.rs | 53 ++++++++++-------- crates/djls-server/src/workspace/store.rs | 57 +++++++++++--------- crates/djls-workspace/src/bridge.rs | 15 ++---- crates/djls-workspace/src/db.rs | 29 +++------- crates/djls-workspace/src/lib.rs | 3 +- crates/djls-workspace/src/watcher.rs | 1 - 6 files changed, 77 insertions(+), 81 deletions(-) diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs index 5a77a65..75812b8 100644 --- a/crates/djls-server/src/workspace/document.rs +++ b/crates/djls-server/src/workspace/document.rs @@ -1,8 +1,7 @@ +use djls_workspace::{FileId, VfsSnapshot}; use std::sync::Arc; use tower_lsp_server::lsp_types::{Position, Range}; -use djls_workspace::{FileId, VfsSnapshot}; -/// Document metadata container - no longer a Salsa input, just plain data #[derive(Clone, Debug)] pub struct TextDocument { pub uri: String, @@ -20,39 +19,50 @@ impl TextDocument { file_id, } } - + pub fn file_id(&self) -> FileId { self.file_id } - + pub fn get_content(&self, vfs: &VfsSnapshot) -> Option> { vfs.get_text(self.file_id) } - + pub fn get_line(&self, vfs: &VfsSnapshot, line_index: &LineIndex, line: u32) -> Option { let content = self.get_content(vfs)?; - + let line_start = *line_index.line_starts.get(line as usize)?; - let line_end = line_index.line_starts + let line_end = line_index + .line_starts .get(line as usize + 1) .copied() .unwrap_or(line_index.length); - + Some(content[line_start as usize..line_end as usize].to_string()) } - - pub fn get_text_range(&self, vfs: &VfsSnapshot, line_index: &LineIndex, range: Range) -> Option { + + pub fn get_text_range( + &self, + vfs: &VfsSnapshot, + line_index: &LineIndex, + range: Range, + ) -> Option { let content = self.get_content(vfs)?; - + let start_offset = line_index.offset(range.start)? as usize; let end_offset = line_index.offset(range.end)? as usize; - + Some(content[start_offset..end_offset].to_string()) } - - pub fn get_template_tag_context(&self, vfs: &VfsSnapshot, line_index: &LineIndex, position: Position) -> Option { + + pub fn get_template_tag_context( + &self, + vfs: &VfsSnapshot, + line_index: &LineIndex, + position: Position, + ) -> Option { let content = self.get_content(vfs)?; - + let start = line_index.line_starts.get(position.line as usize)?; let end = line_index .line_starts @@ -136,16 +146,18 @@ impl LineIndex { } // Find the line text - let next_line_start = self.line_starts.get(position.line as usize + 1) + let next_line_start = self + .line_starts + .get(position.line as usize + 1) .copied() .unwrap_or(self.length); - + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; - + // Convert UTF-16 character offset to UTF-8 byte offset within the line let mut utf16_pos = 0; let mut utf8_pos = 0; - + for c in line_text.chars() { if utf16_pos >= position.character { break; @@ -153,7 +165,7 @@ impl LineIndex { utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); } - + Some(line_start_utf8 + utf8_pos) } @@ -217,4 +229,3 @@ pub struct TemplateTagContext { pub closing_brace: ClosingBrace, pub needs_leading_space: bool, } - diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-server/src/workspace/store.rs index 5a38d37..9096201 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-server/src/workspace/store.rs @@ -110,7 +110,8 @@ impl Store { .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; // Apply text changes using the new function - let new_content = apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; + let new_content = + apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; // Update TextDocument version if let Some(document) = self.documents.get_mut(&uri_str) { @@ -317,11 +318,11 @@ fn apply_text_changes( (Some(range_a), Some(range_b)) => { // Primary sort: by line (reverse) let line_cmp = range_b.start.line.cmp(&range_a.start.line); - if line_cmp != std::cmp::Ordering::Equal { - line_cmp - } else { + if line_cmp == std::cmp::Ordering::Equal { // Secondary sort: by character (reverse) range_b.start.character.cmp(&range_a.start.character) + } else { + line_cmp } } _ => std::cmp::Ordering::Equal, @@ -333,14 +334,20 @@ fn apply_text_changes( for change in &sorted_changes { if let Some(range) = change.range { // Convert UTF-16 positions to UTF-8 offsets - let start_offset = line_index.offset_utf16(range.start, &result) + let start_offset = line_index + .offset_utf16(range.start, &result) .ok_or_else(|| anyhow!("Invalid start position: {:?}", range.start))?; - let end_offset = line_index.offset_utf16(range.end, &result) + let end_offset = line_index + .offset_utf16(range.end, &result) .ok_or_else(|| anyhow!("Invalid end position: {:?}", range.end))?; if start_offset as usize > result.len() || end_offset as usize > result.len() { - return Err(anyhow!("Offset out of bounds: start={}, end={}, len={}", - start_offset, end_offset, result.len())); + return Err(anyhow!( + "Offset out of bounds: start={}, end={}, len={}", + start_offset, + end_offset, + result.len() + )); } // Apply the change @@ -360,7 +367,7 @@ mod tests { fn test_apply_single_character_insertion() { let content = "Hello world"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 6), Position::new(0, 6))), range_length: None, @@ -375,11 +382,11 @@ mod tests { fn test_apply_single_character_deletion() { let content = "Hello world"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 5), Position::new(0, 6))), range_length: None, - text: "".to_string(), + text: String::new(), }]; let result = apply_text_changes(content, &changes, &line_index).unwrap(); @@ -390,7 +397,7 @@ mod tests { fn test_apply_multiple_changes_in_reverse_order() { let content = "line 1\nline 2\nline 3"; let line_index = LineIndex::new(content); - + // Insert "new " at position (1, 0) and "another " at position (0, 0) let changes = vec![ TextDocumentContentChangeEvent { @@ -413,7 +420,7 @@ mod tests { fn test_apply_multiline_replacement() { let content = "line 1\nline 2\nline 3"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 0), Position::new(2, 6))), range_length: None, @@ -428,7 +435,7 @@ mod tests { fn test_apply_full_document_replacement() { let content = "old content"; let line_index = LineIndex::new(content); - + let changes = vec![TextDocumentContentChangeEvent { range: None, range_length: None, @@ -443,7 +450,7 @@ mod tests { fn test_utf16_line_index_basic() { let content = "hello world"; let line_index = LineIndex::new(content); - + // ASCII characters should have 1:1 UTF-8:UTF-16 mapping let pos = Position::new(0, 6); let offset = line_index.offset_utf16(pos, content).unwrap(); @@ -455,11 +462,11 @@ mod tests { fn test_utf16_line_index_with_emoji() { let content = "hello 👋 world"; let line_index = LineIndex::new(content); - + // 👋 is 2 UTF-16 code units but 4 UTF-8 bytes let pos_after_emoji = Position::new(0, 8); // UTF-16 position after "hello 👋" let offset = line_index.offset_utf16(pos_after_emoji, content).unwrap(); - + // Should point to the space before "world" assert_eq!(offset, 10); // UTF-8 byte offset assert_eq!(&content[10..11], " "); @@ -469,7 +476,7 @@ mod tests { fn test_utf16_line_index_multiline() { let content = "first line\nsecond line"; let line_index = LineIndex::new(content); - + let pos = Position::new(1, 7); // Position at 'l' in "line" on second line let offset = line_index.offset_utf16(pos, content).unwrap(); assert_eq!(offset, 18); // 11 (first line + \n) + 7 @@ -480,7 +487,7 @@ mod tests { fn test_apply_changes_with_emoji() { let content = "hello 👋 world"; let line_index = LineIndex::new(content); - + // Insert text after the space following the emoji (UTF-16 position 9) let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 9), Position::new(0, 9))), @@ -496,28 +503,28 @@ mod tests { fn test_line_index_utf16_tracking() { let content = "a👋b"; let line_index = LineIndex::new(content); - + // Check UTF-16 line starts are tracked correctly assert_eq!(line_index.line_starts_utf16, vec![0]); assert_eq!(line_index.length_utf16, 4); // 'a' (1) + 👋 (2) + 'b' (1) = 4 UTF-16 units assert_eq!(line_index.length, 6); // 'a' (1) + 👋 (4) + 'b' (1) = 6 UTF-8 bytes } - #[test] + #[test] fn test_edge_case_changes_at_boundaries() { let content = "abc"; let line_index = LineIndex::new(content); - + // Insert at beginning let changes = vec![TextDocumentContentChangeEvent { range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), range_length: None, text: "start".to_string(), }]; - + let result = apply_text_changes(content, &changes, &line_index).unwrap(); assert_eq!(result, "startabc"); - + // Insert at end let line_index = LineIndex::new(content); let changes = vec![TextDocumentContentChangeEvent { @@ -525,7 +532,7 @@ mod tests { range_length: None, text: "end".to_string(), }]; - + let result = apply_text_changes(content, &changes, &line_index).unwrap(); assert_eq!(result, "abcend"); } diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 78669ad..767afbe 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -9,10 +9,7 @@ use std::{collections::HashMap, sync::Arc}; use salsa::Setter; use super::{ - db::{ - parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, - TemplateLoaderOrder, - }, + db::{parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder}, vfs::{FileKind, VfsSnapshot}, FileId, }; @@ -69,16 +66,12 @@ impl FileStore { pub fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { for (id, rec) in &snap.files { let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); - let new_kind = match rec.meta.kind { - FileKind::Python => FileKindMini::Python, - FileKind::Template => FileKindMini::Template, - FileKind::Other => FileKindMini::Other, - }; + let new_kind = rec.meta.kind; if let Some(sf) = self.files.get(id) { // Update if changed — avoid touching Salsa when not needed if sf.kind(&self.db) != new_kind { - sf.set_kind(&mut self.db).to(new_kind.clone()); + sf.set_kind(&mut self.db).to(new_kind); } if sf.text(&self.db).as_ref() != &*new_text { sf.set_text(&mut self.db).to(new_text.clone()); @@ -100,7 +93,7 @@ impl FileStore { /// Get the file kind classification by its [`FileId`]. /// /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub fn file_kind(&self, id: FileId) -> Option { + pub fn file_kind(&self, id: FileId) -> Option { self.files.get(&id).map(|sf| sf.kind(&self.db)) } diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index a38757d..1b4ece5 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -7,6 +7,8 @@ use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; +use crate::vfs::FileKind; + /// Salsa database root for workspace /// /// The [`Database`] provides default storage and, in tests, captures Salsa events for @@ -49,21 +51,6 @@ impl Default for Database { #[salsa::db] impl salsa::Database for Database {} -/// Minimal classification for analysis routing. -/// -/// [`FileKindMini`] provides a lightweight categorization of files to determine which -/// analysis pipelines should process them. This is the Salsa-side representation -/// of file types, mapped from the VFS layer's `vfs::FileKind`. -#[derive(Clone, Eq, PartialEq, Hash, Debug)] -pub enum FileKindMini { - /// Python source file (.py) - Python, - /// Django template file (.html, .jinja, etc.) - Template, - /// Other file types not requiring specialized analysis - Other, -} - /// Represents a single file's classification and current content. /// /// [`SourceFile`] is a Salsa input entity that tracks both the file's type (for routing @@ -72,7 +59,7 @@ pub enum FileKindMini { #[salsa::input] pub struct SourceFile { /// The file's classification for analysis routing - pub kind: FileKindMini, + pub kind: FileKind, /// The current text content of the file #[returns(ref)] pub text: Arc, @@ -113,7 +100,7 @@ pub struct TemplateAst { #[salsa::tracked] pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option> { // Only parse template files - if file.kind(db) != FileKindMini::Template { + if file.kind(db) != FileKind::Template { return None; } @@ -161,7 +148,7 @@ mod tests { // Create a template file let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); - let file = SourceFile::new(&db, FileKindMini::Template, template_content.clone()); + let file = SourceFile::new(&db, FileKind::Template, template_content.clone()); // First parse - should execute the parsing let ast1 = parse_template(&db, file); @@ -181,7 +168,7 @@ mod tests { // Create a template file let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); - let file = SourceFile::new(&db, FileKindMini::Template, template_content1); + let file = SourceFile::new(&db, FileKind::Template, template_content1); // First parse let ast1 = parse_template(&db, file); @@ -206,7 +193,7 @@ mod tests { // Create a Python file let python_content: Arc = Arc::from("def hello():\n print('Hello')"); - let file = SourceFile::new(&db, FileKindMini::Python, python_content); + let file = SourceFile::new(&db, FileKind::Python, python_content); // Should return None for non-template files let ast = parse_template(&db, file); @@ -223,7 +210,7 @@ mod tests { // Create a template with an error (unclosed tag) let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); - let file = SourceFile::new(&db, FileKindMini::Template, template_content); + let file = SourceFile::new(&db, FileKind::Template, template_content); // Get errors let errors1 = template_errors(&db, file); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 30c69a7..a3db4c3 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -5,8 +5,7 @@ mod watcher; pub use bridge::FileStore; pub use db::{ - parse_template, template_errors, Database, FileKindMini, SourceFile, TemplateAst, - TemplateLoaderOrder, + parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder, }; pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/watcher.rs index 57c798c..55fa9a7 100644 --- a/crates/djls-workspace/src/watcher.rs +++ b/crates/djls-workspace/src/watcher.rs @@ -317,4 +317,3 @@ mod tests { assert_ne!(deleted, renamed); } } - From 3131470cce738d8e2220a4ef66e6d25cf8256e2c Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 12:22:54 -0500 Subject: [PATCH 09/30] wip --- Cargo.lock | 2 + crates/djls-server/src/lib.rs | 1 - crates/djls-server/src/server.rs | 103 ++++++- crates/djls-server/src/session.rs | 55 +++- crates/djls-server/src/workspace/document.rs | 231 -------------- crates/djls-server/src/workspace/mod.rs | 6 - crates/djls-server/src/workspace/utils.rs | 43 --- crates/djls-templates/src/lib.rs | 2 +- crates/djls-workspace/Cargo.toml | 2 + crates/djls-workspace/src/bridge.rs | 33 +- crates/djls-workspace/src/db.rs | 5 +- .../djls-workspace/src/document/language.rs | 34 +++ .../djls-workspace/src/document/line_index.rs | 87 ++++++ crates/djls-workspace/src/document/mod.rs | 132 ++++++++ .../src/document}/store.rs | 289 ++++++++++++++---- .../djls-workspace/src/document/template.rs | 13 + crates/djls-workspace/src/lib.rs | 16 +- .../djls-workspace/src/{vfs.rs => vfs/mod.rs} | 115 ++++--- .../djls-workspace/src/{ => vfs}/watcher.rs | 22 +- 19 files changed, 761 insertions(+), 430 deletions(-) delete mode 100644 crates/djls-server/src/workspace/document.rs delete mode 100644 crates/djls-server/src/workspace/mod.rs delete mode 100644 crates/djls-server/src/workspace/utils.rs create mode 100644 crates/djls-workspace/src/document/language.rs create mode 100644 crates/djls-workspace/src/document/line_index.rs create mode 100644 crates/djls-workspace/src/document/mod.rs rename crates/{djls-server/src/workspace => djls-workspace/src/document}/store.rs (67%) create mode 100644 crates/djls-workspace/src/document/template.rs rename crates/djls-workspace/src/{vfs.rs => vfs/mod.rs} (84%) rename crates/djls-workspace/src/{ => vfs}/watcher.rs (97%) diff --git a/Cargo.lock b/Cargo.lock index 73c6aae..d739a68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,11 +509,13 @@ dependencies = [ "anyhow", "camino", "dashmap", + "djls-project", "djls-templates", "notify", "salsa", "tempfile", "tokio", + "tower-lsp-server", "url", ] diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index 1ebe618..b601c7a 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -3,7 +3,6 @@ mod logging; mod queue; mod server; mod session; -mod workspace; use std::io::IsTerminal; diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index 9df7c63..e61bb08 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -218,8 +218,23 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - if let Err(e) = session.documents_mut().handle_did_open(¶ms) { - tracing::error!("Failed to handle did_open: {}", e); + let uri = params.text_document.uri.clone(); + let version = params.text_document.version; + let language_id = + djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); + let text = params.text_document.text.clone(); + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + if let Err(e) = + session + .documents_mut() + .open_document(url, version, language_id, text) + { + tracing::error!("Failed to handle did_open: {}", e); + } + } else { + tracing::error!("Invalid URI: {:?}", uri); } }) .await; @@ -229,7 +244,21 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - let _ = session.documents_mut().handle_did_change(¶ms); + let uri = ¶ms.text_document.uri; + let version = params.text_document.version; + let changes = params.content_changes.clone(); + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + if let Err(e) = session + .documents_mut() + .update_document(&url, version, changes) + { + tracing::error!("Failed to handle did_change: {}", e); + } + } else { + tracing::error!("Invalid URI: {:?}", uri); + } }) .await; } @@ -238,7 +267,14 @@ impl LanguageServer for DjangoLanguageServer { tracing::info!("Closed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { - session.documents_mut().handle_did_close(¶ms); + let uri = ¶ms.text_document.uri; + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + session.documents_mut().close_document(&url); + } else { + tracing::error!("Invalid URI: {:?}", uri); + } }) .await; } @@ -248,14 +284,61 @@ impl LanguageServer for DjangoLanguageServer { .with_session(|session| { if let Some(project) = session.project() { if let Some(tags) = project.template_tags() { - return session.documents().get_completions( - params.text_document_position.text_document.uri.as_str(), - params.text_document_position.position, - tags, - ); + let uri = ¶ms.text_document_position.text_document.uri; + let position = params.text_document_position.position; + + // Convert LSP Uri to url::Url + if let Ok(url) = url::Url::parse(&uri.to_string()) { + if let Some(context) = session.documents().get_template_context(&url, position) { + // Use the context to generate completions + let mut completions: Vec = tags + .iter() + .filter(|tag| { + context.partial_tag.is_empty() || tag.name().starts_with(&context.partial_tag) + }) + .map(|tag| { + let leading_space = if context.needs_leading_space { " " } else { "" }; + tower_lsp_server::lsp_types::CompletionItem { + label: tag.name().to_string(), + kind: Some(tower_lsp_server::lsp_types::CompletionItemKind::KEYWORD), + detail: Some(format!("Template tag from {}", tag.library())), + documentation: tag.doc().as_ref().map(|doc| { + tower_lsp_server::lsp_types::Documentation::MarkupContent( + tower_lsp_server::lsp_types::MarkupContent { + kind: tower_lsp_server::lsp_types::MarkupKind::Markdown, + value: (*doc).to_string(), + } + ) + }), + insert_text: Some(match context.closing_brace { + djls_workspace::ClosingBrace::None => format!("{}{} %}}", leading_space, tag.name()), + djls_workspace::ClosingBrace::PartialClose => format!("{}{} %", leading_space, tag.name()), + djls_workspace::ClosingBrace::FullClose => format!("{}{} ", leading_space, tag.name()), + }), + insert_text_format: Some(tower_lsp_server::lsp_types::InsertTextFormat::PLAIN_TEXT), + ..Default::default() + } + }) + .collect(); + + if completions.is_empty() { + None + } else { + completions.sort_by(|a, b| a.label.cmp(&b.label)); + Some(tower_lsp_server::lsp_types::CompletionResponse::Array(completions)) + } + } else { + None + } + } else { + None + } + } else { + None } + } else { + None } - None }) .await) } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index a5832cb..8a584d3 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,14 +1,17 @@ +use std::path::PathBuf; + use djls_conf::Settings; use djls_project::DjangoProject; +use djls_workspace::DocumentStore; +use percent_encoding::percent_decode_str; use tower_lsp_server::lsp_types::ClientCapabilities; use tower_lsp_server::lsp_types::InitializeParams; - -use crate::workspace::Store; +use tower_lsp_server::lsp_types::Uri; #[derive(Default)] pub struct Session { project: Option, - documents: Store, + documents: DocumentStore, settings: Settings, #[allow(dead_code)] @@ -16,8 +19,46 @@ pub struct Session { } impl Session { + /// Determines the project root path from initialization parameters. + /// + /// Tries the current directory first, then falls back to the first workspace folder. + fn get_project_path(params: &InitializeParams) -> Option { + // Try current directory first + std::env::current_dir().ok().or_else(|| { + // Fall back to the first workspace folder URI + params + .workspace_folders + .as_ref() + .and_then(|folders| folders.first()) + .and_then(|folder| Self::uri_to_pathbuf(&folder.uri)) + }) + } + + /// Converts a `file:` URI into an absolute `PathBuf`. + fn uri_to_pathbuf(uri: &Uri) -> Option { + // Check if the scheme is "file" + if uri.scheme().is_none_or(|s| s.as_str() != "file") { + return None; + } + + // Get the path part as a string + let encoded_path_str = uri.path().as_str(); + + // Decode the percent-encoded path string + let decoded_path_cow = percent_decode_str(encoded_path_str).decode_utf8_lossy(); + let path_str = decoded_path_cow.as_ref(); + + #[cfg(windows)] + let path_str = { + // Remove leading '/' for paths like /C:/... + path_str.strip_prefix('/').unwrap_or(path_str) + }; + + Some(PathBuf::from(path_str)) + } + pub fn new(params: &InitializeParams) -> Self { - let project_path = crate::workspace::get_project_path(params); + let project_path = Self::get_project_path(params); let (project, settings) = if let Some(path) = &project_path { let settings = @@ -33,7 +74,7 @@ impl Session { Self { client_capabilities: params.capabilities.clone(), project, - documents: Store::default(), + documents: DocumentStore::new(), settings, } } @@ -46,11 +87,11 @@ impl Session { &mut self.project } - pub fn documents(&self) -> &Store { + pub fn documents(&self) -> &DocumentStore { &self.documents } - pub fn documents_mut(&mut self) -> &mut Store { + pub fn documents_mut(&mut self) -> &mut DocumentStore { &mut self.documents } diff --git a/crates/djls-server/src/workspace/document.rs b/crates/djls-server/src/workspace/document.rs deleted file mode 100644 index 75812b8..0000000 --- a/crates/djls-server/src/workspace/document.rs +++ /dev/null @@ -1,231 +0,0 @@ -use djls_workspace::{FileId, VfsSnapshot}; -use std::sync::Arc; -use tower_lsp_server::lsp_types::{Position, Range}; - -#[derive(Clone, Debug)] -pub struct TextDocument { - pub uri: String, - pub version: i32, - pub language_id: LanguageId, - file_id: FileId, -} - -impl TextDocument { - pub fn new(uri: String, version: i32, language_id: LanguageId, file_id: FileId) -> Self { - Self { - uri, - version, - language_id, - file_id, - } - } - - pub fn file_id(&self) -> FileId { - self.file_id - } - - pub fn get_content(&self, vfs: &VfsSnapshot) -> Option> { - vfs.get_text(self.file_id) - } - - pub fn get_line(&self, vfs: &VfsSnapshot, line_index: &LineIndex, line: u32) -> Option { - let content = self.get_content(vfs)?; - - let line_start = *line_index.line_starts.get(line as usize)?; - let line_end = line_index - .line_starts - .get(line as usize + 1) - .copied() - .unwrap_or(line_index.length); - - Some(content[line_start as usize..line_end as usize].to_string()) - } - - pub fn get_text_range( - &self, - vfs: &VfsSnapshot, - line_index: &LineIndex, - range: Range, - ) -> Option { - let content = self.get_content(vfs)?; - - let start_offset = line_index.offset(range.start)? as usize; - let end_offset = line_index.offset(range.end)? as usize; - - Some(content[start_offset..end_offset].to_string()) - } - - pub fn get_template_tag_context( - &self, - vfs: &VfsSnapshot, - line_index: &LineIndex, - position: Position, - ) -> Option { - let content = self.get_content(vfs)?; - - let start = line_index.line_starts.get(position.line as usize)?; - let end = line_index - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(line_index.length); - - let line = &content[*start as usize..end as usize]; - let char_pos: usize = position.character.try_into().ok()?; - let prefix = &line[..char_pos]; - let rest_of_line = &line[char_pos..]; - let rest_trimmed = rest_of_line.trim_start(); - - prefix.rfind("{%").map(|tag_start| { - // Check if we're immediately after {% with no space - let needs_leading_space = prefix.ends_with("{%"); - - let closing_brace = if rest_trimmed.starts_with("%}") { - ClosingBrace::FullClose - } else if rest_trimmed.starts_with('}') { - ClosingBrace::PartialClose - } else { - ClosingBrace::None - }; - - TemplateTagContext { - partial_tag: prefix[tag_start + 2..].trim().to_string(), - closing_brace, - needs_leading_space, - } - }) - } -} - -#[derive(Clone, Debug)] -pub struct LineIndex { - pub line_starts: Vec, - pub line_starts_utf16: Vec, - pub length: u32, - pub length_utf16: u32, -} - -impl LineIndex { - pub fn new(text: &str) -> Self { - let mut line_starts = vec![0]; - let mut line_starts_utf16 = vec![0]; - let mut pos_utf8 = 0; - let mut pos_utf16 = 0; - - for c in text.chars() { - pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); - pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); - if c == '\n' { - line_starts.push(pos_utf8); - line_starts_utf16.push(pos_utf16); - } - } - - Self { - line_starts, - line_starts_utf16, - length: pos_utf8, - length_utf16: pos_utf16, - } - } - - pub fn offset(&self, position: Position) -> Option { - let line_start = self.line_starts.get(position.line as usize)?; - - Some(line_start + position.character) - } - - /// Convert UTF-16 LSP position to UTF-8 byte offset - pub fn offset_utf16(&self, position: Position, text: &str) -> Option { - let line_start_utf8 = self.line_starts.get(position.line as usize)?; - let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; - - // If position is at start of line, return UTF-8 line start - if position.character == 0 { - return Some(*line_start_utf8); - } - - // Find the line text - let next_line_start = self - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.length); - - let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; - - // Convert UTF-16 character offset to UTF-8 byte offset within the line - let mut utf16_pos = 0; - let mut utf8_pos = 0; - - for c in line_text.chars() { - if utf16_pos >= position.character { - break; - } - utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); - utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); - } - - Some(line_start_utf8 + utf8_pos) - } - - #[allow(dead_code)] - pub fn position(&self, offset: u32) -> Position { - let line = match self.line_starts.binary_search(&offset) { - Ok(line) => line, - Err(line) => line - 1, - }; - - let line_start = self.line_starts[line]; - let character = offset - line_start; - - Position::new(u32::try_from(line).unwrap_or(0), character) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum LanguageId { - HtmlDjango, - Other, - Python, -} - -impl From<&str> for LanguageId { - fn from(language_id: &str) -> Self { - match language_id { - "django-html" | "htmldjango" => Self::HtmlDjango, - "python" => Self::Python, - _ => Self::Other, - } - } -} - -impl From for LanguageId { - fn from(language_id: String) -> Self { - Self::from(language_id.as_str()) - } -} - -impl From for djls_workspace::FileKind { - fn from(language_id: LanguageId) -> Self { - match language_id { - LanguageId::Python => Self::Python, - LanguageId::HtmlDjango => Self::Template, - LanguageId::Other => Self::Other, - } - } -} - -#[derive(Debug)] -pub enum ClosingBrace { - None, - PartialClose, // just } - FullClose, // %} -} - -#[derive(Debug)] -pub struct TemplateTagContext { - pub partial_tag: String, - pub closing_brace: ClosingBrace, - pub needs_leading_space: bool, -} diff --git a/crates/djls-server/src/workspace/mod.rs b/crates/djls-server/src/workspace/mod.rs deleted file mode 100644 index fb15df9..0000000 --- a/crates/djls-server/src/workspace/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -mod document; -mod store; -mod utils; - -pub use store::Store; -pub use utils::get_project_path; diff --git a/crates/djls-server/src/workspace/utils.rs b/crates/djls-server/src/workspace/utils.rs deleted file mode 100644 index 08a40ba..0000000 --- a/crates/djls-server/src/workspace/utils.rs +++ /dev/null @@ -1,43 +0,0 @@ -use std::path::PathBuf; - -use percent_encoding::percent_decode_str; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::Uri; - -/// Determines the project root path from initialization parameters. -/// -/// Tries the current directory first, then falls back to the first workspace folder. -pub fn get_project_path(params: &InitializeParams) -> Option { - // Try current directory first - std::env::current_dir().ok().or_else(|| { - // Fall back to the first workspace folder URI - params - .workspace_folders - .as_ref() - .and_then(|folders| folders.first()) - .and_then(|folder| uri_to_pathbuf(&folder.uri)) - }) -} - -/// Converts a `file:` URI into an absolute `PathBuf`. -fn uri_to_pathbuf(uri: &Uri) -> Option { - // Check if the scheme is "file" - if uri.scheme().is_none_or(|s| s.as_str() != "file") { - return None; - } - - // Get the path part as a string - let encoded_path_str = uri.path().as_str(); - - // Decode the percent-encoded path string - let decoded_path_cow = percent_decode_str(encoded_path_str).decode_utf8_lossy(); - let path_str = decoded_path_cow.as_ref(); - - #[cfg(windows)] - let path_str = { - // Remove leading '/' for paths like /C:/... - path_str.strip_prefix('/').unwrap_or(path_str) - }; - - Some(PathBuf::from(path_str)) -} diff --git a/crates/djls-templates/src/lib.rs b/crates/djls-templates/src/lib.rs index 7eab1f6..7c2369c 100644 --- a/crates/djls-templates/src/lib.rs +++ b/crates/djls-templates/src/lib.rs @@ -1,4 +1,4 @@ -mod ast; +pub mod ast; mod error; mod lexer; mod parser; diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 34fa01e..0a46bd8 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] djls-templates = { workspace = true } +djls-project = { workspace = true } anyhow = { workspace = true } camino = { workspace = true } @@ -12,6 +13,7 @@ dashmap = { workspace = true } notify = { workspace = true } salsa = { workspace = true } tokio = { workspace = true } +tower-lsp-server = { workspace = true } url = { workspace = true } [dev-dependencies] diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 767afbe..0338262 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -4,15 +4,20 @@ //! It ensures we only touch Salsa when content or classification changes, maximizing //! incremental performance. -use std::{collections::HashMap, sync::Arc}; +use std::collections::HashMap; +use std::sync::Arc; use salsa::Setter; -use super::{ - db::{parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder}, - vfs::{FileKind, VfsSnapshot}, - FileId, -}; +use super::db::parse_template; +use super::db::template_errors; +use super::db::Database; +use super::db::SourceFile; +use super::db::TemplateAst; +use super::db::TemplateLoaderOrder; +use super::vfs::FileKind; +use super::vfs::VfsSnapshot; +use super::FileId; /// Owner of the Salsa [`Database`] plus the handles for updating inputs. /// @@ -63,7 +68,7 @@ impl FileStore { /// /// The method is idempotent and minimizes Salsa invalidations by checking for /// actual changes before updating inputs. - pub fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { + pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { for (id, rec) in &snap.files { let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); let new_kind = rec.meta.kind; @@ -86,14 +91,14 @@ impl FileStore { /// Get the text content of a file by its [`FileId`]. /// /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub fn file_text(&self, id: FileId) -> Option> { + pub(crate) fn file_text(&self, id: FileId) -> Option> { self.files.get(&id).map(|sf| sf.text(&self.db).clone()) } /// Get the file kind classification by its [`FileId`]. /// /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub fn file_kind(&self, id: FileId) -> Option { + pub(crate) fn file_kind(&self, id: FileId) -> Option { self.files.get(&id).map(|sf| sf.kind(&self.db)) } @@ -102,7 +107,7 @@ impl FileStore { /// This method leverages Salsa's incremental computation to cache parsed ASTs. /// The AST is only re-parsed when the file's content changes in the VFS. /// Returns `None` if the file is not tracked or is not a template file. - pub fn get_template_ast(&self, id: FileId) -> Option> { + pub(crate) fn get_template_ast(&self, id: FileId) -> Option> { let source_file = self.files.get(&id)?; parse_template(&self.db, *source_file) } @@ -112,7 +117,7 @@ impl FileStore { /// This method provides quick access to template errors without needing the full AST. /// Useful for diagnostics and error reporting. Returns an empty slice for /// non-template files or files not tracked in the store. - pub fn get_template_errors(&self, id: FileId) -> Arc<[String]> { + pub(crate) fn get_template_errors(&self, id: FileId) -> Arc<[String]> { self.files .get(&id) .map_or_else(|| Arc::from(vec![]), |sf| template_errors(&self.db, *sf)) @@ -127,10 +132,12 @@ impl Default for FileStore { #[cfg(test)] mod tests { - use super::*; - use crate::vfs::{TextSource, Vfs}; use camino::Utf8PathBuf; + use super::*; + use crate::vfs::TextSource; + use crate::vfs::Vfs; + #[test] fn test_filestore_template_ast_caching() { let mut store = FileStore::new(); diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 1b4ece5..2783eae 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -7,6 +7,8 @@ use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; +use djls_templates::Ast; + use crate::vfs::FileKind; /// Salsa database root for workspace @@ -139,9 +141,10 @@ pub fn template_errors(db: &dyn salsa::Database, file: SourceFile) -> Arc<[Strin #[cfg(test)] mod tests { - use super::*; use salsa::Setter; + use super::*; + #[test] fn test_template_parsing_caches_result() { let db = Database::default(); diff --git a/crates/djls-workspace/src/document/language.rs b/crates/djls-workspace/src/document/language.rs new file mode 100644 index 0000000..09f0bb5 --- /dev/null +++ b/crates/djls-workspace/src/document/language.rs @@ -0,0 +1,34 @@ +use crate::vfs::FileKind; + +#[derive(Clone, Debug, PartialEq)] +pub enum LanguageId { + HtmlDjango, + Other, + Python, +} + +impl From<&str> for LanguageId { + fn from(language_id: &str) -> Self { + match language_id { + "django-html" | "htmldjango" => Self::HtmlDjango, + "python" => Self::Python, + _ => Self::Other, + } + } +} + +impl From for LanguageId { + fn from(language_id: String) -> Self { + Self::from(language_id.as_str()) + } +} + +impl From for FileKind { + fn from(language_id: LanguageId) -> Self { + match language_id { + LanguageId::Python => Self::Python, + LanguageId::HtmlDjango => Self::Template, + LanguageId::Other => Self::Other, + } + } +} diff --git a/crates/djls-workspace/src/document/line_index.rs b/crates/djls-workspace/src/document/line_index.rs new file mode 100644 index 0000000..39f1fde --- /dev/null +++ b/crates/djls-workspace/src/document/line_index.rs @@ -0,0 +1,87 @@ +use tower_lsp_server::lsp_types::Position; + +#[derive(Clone, Debug)] +pub struct LineIndex { + pub line_starts: Vec, + pub line_starts_utf16: Vec, + pub length: u32, + pub length_utf16: u32, +} + +impl LineIndex { + pub fn new(text: &str) -> Self { + let mut line_starts = vec![0]; + let mut line_starts_utf16 = vec![0]; + let mut pos_utf8 = 0; + let mut pos_utf16 = 0; + + for c in text.chars() { + pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); + if c == '\n' { + line_starts.push(pos_utf8); + line_starts_utf16.push(pos_utf16); + } + } + + Self { + line_starts, + line_starts_utf16, + length: pos_utf8, + length_utf16: pos_utf16, + } + } + + pub fn offset(&self, position: Position) -> Option { + let line_start = self.line_starts.get(position.line as usize)?; + + Some(line_start + position.character) + } + + /// Convert UTF-16 LSP position to UTF-8 byte offset + pub fn offset_utf16(&self, position: Position, text: &str) -> Option { + let line_start_utf8 = self.line_starts.get(position.line as usize)?; + let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + + // If position is at start of line, return UTF-8 line start + if position.character == 0 { + return Some(*line_start_utf8); + } + + // Find the line text + let next_line_start = self + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.length); + + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + + // Convert UTF-16 character offset to UTF-8 byte offset within the line + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); + } + + Some(line_start_utf8 + utf8_pos) + } + + #[allow(dead_code)] + pub fn position(&self, offset: u32) -> Position { + let line = match self.line_starts.binary_search(&offset) { + Ok(line) => line, + Err(line) => line - 1, + }; + + let line_start = self.line_starts[line]; + let character = offset - line_start; + + Position::new(u32::try_from(line).unwrap_or(0), character) + } +} diff --git a/crates/djls-workspace/src/document/mod.rs b/crates/djls-workspace/src/document/mod.rs new file mode 100644 index 0000000..840cd17 --- /dev/null +++ b/crates/djls-workspace/src/document/mod.rs @@ -0,0 +1,132 @@ +mod language; +mod line_index; +mod store; +mod template; + +pub use language::LanguageId; +pub use line_index::LineIndex; +pub use store::DocumentStore; +pub use template::ClosingBrace; +pub use template::TemplateTagContext; +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; + +use crate::FileId; + +#[derive(Clone, Debug)] +pub struct TextDocument { + pub uri: String, + pub version: i32, + pub language_id: LanguageId, + pub(crate) file_id: FileId, + line_index: LineIndex, +} + +impl TextDocument { + pub(crate) fn new( + uri: String, + version: i32, + language_id: LanguageId, + file_id: FileId, + content: &str, + ) -> Self { + let line_index = LineIndex::new(content); + Self { + uri, + version, + language_id, + file_id, + line_index, + } + } + + pub(crate) fn file_id(&self) -> FileId { + self.file_id + } + + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + pub fn get_content<'a>(&self, content: &'a str) -> &'a str { + content + } + + pub fn get_line(&self, content: &str, line: u32) -> Option { + let line_start = *self.line_index.line_starts.get(line as usize)?; + let line_end = self + .line_index + .line_starts + .get(line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + Some(content[line_start as usize..line_end as usize].to_string()) + } + + pub fn get_text_range(&self, content: &str, range: Range) -> Option { + let start_offset = self.line_index.offset(range.start)? as usize; + let end_offset = self.line_index.offset(range.end)? as usize; + + Some(content[start_offset..end_offset].to_string()) + } + + pub fn get_template_tag_context( + &self, + content: &str, + position: Position, + ) -> Option { + let start = self.line_index.line_starts.get(position.line as usize)?; + let end = self + .line_index + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + let line = &content[*start as usize..end as usize]; + let char_pos: usize = position.character.try_into().ok()?; + let prefix = &line[..char_pos]; + let rest_of_line = &line[char_pos..]; + let rest_trimmed = rest_of_line.trim_start(); + + prefix.rfind("{%").map(|tag_start| { + // Check if we're immediately after {% with no space + let needs_leading_space = prefix.ends_with("{%"); + + let closing_brace = if rest_trimmed.starts_with("%}") { + ClosingBrace::FullClose + } else if rest_trimmed.starts_with('}') { + ClosingBrace::PartialClose + } else { + ClosingBrace::None + }; + + TemplateTagContext { + partial_tag: prefix[tag_start + 2..].trim().to_string(), + closing_brace, + needs_leading_space, + } + }) + } + + pub fn position_to_offset(&self, position: Position) -> Option { + self.line_index.offset(position) + } + + pub fn offset_to_position(&self, offset: u32) -> Position { + self.line_index.position(offset) + } + + pub fn update_content(&mut self, content: &str) { + self.line_index = LineIndex::new(content); + } + + pub fn version(&self) -> i32 { + self.version + } + + pub fn language_id(&self) -> LanguageId { + self.language_id.clone() + } +} diff --git a/crates/djls-server/src/workspace/store.rs b/crates/djls-workspace/src/document/store.rs similarity index 67% rename from crates/djls-server/src/workspace/store.rs rename to crates/djls-workspace/src/document/store.rs index 9096201..8284107 100644 --- a/crates/djls-server/src/workspace/store.rs +++ b/crates/djls-workspace/src/document/store.rs @@ -1,11 +1,11 @@ use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; +use std::sync::Mutex; use anyhow::anyhow; use anyhow::Result; use camino::Utf8PathBuf; use djls_project::TemplateTags; -use djls_workspace::{FileId, FileKind, FileStore, TextSource, Vfs}; use tower_lsp_server::lsp_types::CompletionItem; use tower_lsp_server::lsp_types::CompletionItemKind; use tower_lsp_server::lsp_types::CompletionResponse; @@ -22,31 +22,118 @@ use tower_lsp_server::lsp_types::Position; use tower_lsp_server::lsp_types::Range; use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; -use super::document::{ClosingBrace, LanguageId, LineIndex, TextDocument}; +use crate::bridge::FileStore; +use crate::db::TemplateAst; +use crate::vfs::FileKind; +use crate::vfs::TextSource; +use crate::vfs::Vfs; +use crate::ClosingBrace; +use crate::LanguageId; +use crate::LineIndex; +use crate::TextDocument; -pub struct Store { +pub struct DocumentStore { vfs: Arc, file_store: Arc>, - file_ids: HashMap, - line_indices: HashMap, - versions: HashMap, documents: HashMap, } -impl Default for Store { +impl Default for DocumentStore { fn default() -> Self { Self { vfs: Arc::new(Vfs::default()), file_store: Arc::new(Mutex::new(FileStore::new())), - file_ids: HashMap::new(), - line_indices: HashMap::new(), - versions: HashMap::new(), documents: HashMap::new(), } } } -impl Store { +impl DocumentStore { + pub fn new() -> Self { + Self::default() + } + + /// Open a document with the given URI, version, language, and text content. + /// This creates a new TextDocument and stores it internally, hiding VFS details. + pub fn open_document( + &mut self, + uri: url::Url, + version: i32, + language_id: LanguageId, + text: String, + ) -> Result<()> { + let uri_str = uri.to_string(); + let kind = FileKind::from(language_id.clone()); + + // Convert URI to path - simplified for now, just use URI string + let path = Utf8PathBuf::from(uri.as_str()); + + // Store content in VFS + let text_source = TextSource::Overlay(Arc::from(text.as_str())); + let file_id = self.vfs.intern_file(uri, path, kind, text_source); + + // Set overlay content in VFS + self.vfs.set_overlay(file_id, Arc::from(text.as_str()))?; + + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); + + // Create TextDocument with LineIndex + let document = TextDocument::new(uri_str.clone(), version, language_id, file_id, &text); + self.documents.insert(uri_str, document); + + Ok(()) + } + + /// Update a document with the given URI, version, and text changes. + /// This applies changes to the document and updates the VFS accordingly. + pub fn update_document( + &mut self, + uri: &url::Url, + version: i32, + changes: Vec, + ) -> Result<()> { + let uri_str = uri.to_string(); + + // Get document and file_id from the documents HashMap + let document = self + .documents + .get(&uri_str) + .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; + let file_id = document.file_id(); + + // Get current content from VFS + let snapshot = self.vfs.snapshot(); + let current_content = snapshot + .get_text(file_id) + .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; + + // Get line index from the document + let line_index = document.line_index(); + + // Apply text changes using the existing function + let new_content = apply_text_changes(¤t_content, &changes, line_index)?; + + // Update TextDocument version and content + if let Some(document) = self.documents.get_mut(&uri_str) { + document.version = version; + document.update_content(&new_content); + } + + // Update VFS with new content + self.vfs + .set_overlay(file_id, Arc::from(new_content.as_str()))?; + + // Sync VFS snapshot to FileStore for Salsa tracking + let snapshot = self.vfs.snapshot(); + let mut file_store = self.file_store.lock().unwrap(); + file_store.apply_vfs_snapshot(&snapshot); + + Ok(()) + } + pub fn handle_did_open(&mut self, params: &DidOpenTextDocumentParams) -> Result<()> { let uri_str = params.text_document.uri.to_string(); let uri = params.text_document.uri.clone(); @@ -75,13 +162,14 @@ impl Store { file_store.apply_vfs_snapshot(&snapshot); // Create TextDocument metadata - let document = TextDocument::new(uri_str.clone(), version, language_id.clone(), file_id); - self.documents.insert(uri_str.clone(), document); - - // Cache mappings and indices - self.file_ids.insert(uri_str.clone(), file_id); - self.line_indices.insert(file_id, LineIndex::new(&content)); - self.versions.insert(uri_str, version); + let document = TextDocument::new( + uri_str.clone(), + version, + language_id.clone(), + file_id, + &content, + ); + self.documents.insert(uri_str, document); Ok(()) } @@ -90,12 +178,12 @@ impl Store { let uri_str = params.text_document.uri.as_str().to_string(); let version = params.text_document.version; - // Look up FileId - let file_id = self - .file_ids + // Get document and file_id from the documents HashMap + let document = self + .documents .get(&uri_str) - .copied() .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; + let file_id = document.file_id(); // Get current content from VFS let snapshot = self.vfs.snapshot(); @@ -103,19 +191,17 @@ impl Store { .get_text(file_id) .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - // Get current line index for position calculations - let line_index = self - .line_indices - .get(&file_id) - .ok_or_else(|| anyhow!("Line index not found for: {}", uri_str))?; + // Get line index from the document (TextDocument now stores its own LineIndex) + let line_index = document.line_index(); // Apply text changes using the new function let new_content = apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; - // Update TextDocument version + // Update TextDocument version and content if let Some(document) = self.documents.get_mut(&uri_str) { document.version = version; + document.update_content(&new_content); } // Update VFS with new content @@ -127,41 +213,38 @@ impl Store { let mut file_store = self.file_store.lock().unwrap(); file_store.apply_vfs_snapshot(&snapshot); - // Update cached line index and version - self.line_indices - .insert(file_id, LineIndex::new(&new_content)); - self.versions.insert(uri_str, version); - Ok(()) } + /// Close a document with the given URI. + /// This removes the document from internal storage and cleans up resources. + pub fn close_document(&mut self, uri: &url::Url) { + let uri_str = uri.as_str(); + + // Remove TextDocument metadata + self.documents.remove(uri_str); + + // Note: We don't remove from VFS as it might be useful for caching + // The VFS will handle cleanup internally + } + pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { let uri_str = params.text_document.uri.as_str(); // Remove TextDocument metadata self.documents.remove(uri_str); - // Look up FileId and remove mappings - if let Some(file_id) = self.file_ids.remove(uri_str) { - self.line_indices.remove(&file_id); - } - self.versions.remove(uri_str); - // Note: We don't remove from VFS as it might be useful for caching // The VFS will handle cleanup internally } - pub fn get_file_id(&self, uri: &str) -> Option { - self.file_ids.get(uri).copied() - } - - pub fn get_line_index(&self, file_id: FileId) -> Option<&LineIndex> { - self.line_indices.get(&file_id) + pub fn get_line_index(&self, uri: &str) -> Option<&LineIndex> { + self.documents.get(uri).map(|doc| doc.line_index()) } #[allow(dead_code)] pub fn get_version(&self, uri: &str) -> Option { - self.versions.get(uri).copied() + self.documents.get(uri).map(|doc| doc.version()) } #[allow(dead_code)] @@ -178,6 +261,99 @@ impl Store { self.documents.get_mut(uri) } + // URI-based query methods (new API) + pub fn get_document_by_url(&self, uri: &url::Url) -> Option<&TextDocument> { + self.get_document(uri.as_str()) + } + + pub fn get_document_text(&self, uri: &url::Url) -> Option> { + let document = self.get_document_by_url(uri)?; + let file_id = document.file_id(); + let snapshot = self.vfs.snapshot(); + snapshot.get_text(file_id) + } + + pub fn get_line_text(&self, uri: &url::Url, line: u32) -> Option { + let document = self.get_document_by_url(uri)?; + let snapshot = self.vfs.snapshot(); + let content = snapshot.get_text(document.file_id())?; + document.get_line(content.as_ref(), line) + } + + pub fn get_word_at_position(&self, uri: &url::Url, position: Position) -> Option { + // This is a simplified implementation - get the line and extract word at position + let line_text = self.get_line_text(uri, position.line)?; + let char_pos: usize = position.character.try_into().ok()?; + + if char_pos >= line_text.len() { + return None; + } + + // Find word boundaries (simplified - considers alphanumeric and underscore as word chars) + let line_bytes = line_text.as_bytes(); + let mut start = char_pos; + let mut end = char_pos; + + // Find start of word + while start > 0 && is_word_char(line_bytes[start - 1]) { + start -= 1; + } + + // Find end of word + while end < line_text.len() && is_word_char(line_bytes[end]) { + end += 1; + } + + if start < end { + Some(line_text[start..end].to_string()) + } else { + None + } + } + + // Position mapping methods + pub fn offset_to_position(&self, uri: &url::Url, offset: usize) -> Option { + let document = self.get_document_by_url(uri)?; + Some(document.offset_to_position(offset as u32)) + } + + pub fn position_to_offset(&self, uri: &url::Url, position: Position) -> Option { + let document = self.get_document_by_url(uri)?; + document + .position_to_offset(position) + .map(|offset| offset as usize) + } + + // Template-specific methods + pub fn get_template_ast(&self, uri: &url::Url) -> Option> { + let document = self.get_document_by_url(uri)?; + let file_id = document.file_id(); + let file_store = self.file_store.lock().unwrap(); + file_store.get_template_ast(file_id) + } + + pub fn get_template_errors(&self, uri: &url::Url) -> Vec { + let document = match self.get_document_by_url(uri) { + Some(doc) => doc, + None => return vec![], + }; + let file_id = document.file_id(); + let file_store = self.file_store.lock().unwrap(); + let errors = file_store.get_template_errors(file_id); + errors.to_vec() + } + + pub fn get_template_context( + &self, + uri: &url::Url, + position: Position, + ) -> Option { + let document = self.get_document_by_url(uri)?; + let snapshot = self.vfs.snapshot(); + let content = snapshot.get_text(document.file_id())?; + document.get_template_tag_context(content.as_ref(), position) + } + pub fn get_completions( &self, uri: &str, @@ -186,7 +362,7 @@ impl Store { ) -> Option { // Check if this is a Django template using TextDocument metadata let document = self.get_document(uri)?; - if document.language_id != LanguageId::HtmlDjango { + if document.language_id() != LanguageId::HtmlDjango { return None; } @@ -202,8 +378,9 @@ impl Store { // Get template tag context from document let vfs_snapshot = self.vfs.snapshot(); - let line_index = self.get_line_index(file_id)?; - let context = document.get_template_tag_context(&vfs_snapshot, line_index, position)?; + let text_content = vfs_snapshot.get_text(file_id)?; + let content = text_content.as_ref(); + let context = document.get_template_tag_context(content, position)?; let mut completions: Vec = tags .iter() @@ -252,12 +429,12 @@ impl Store { }; // Only process template files - if document.language_id != LanguageId::HtmlDjango { + if document.language_id() != LanguageId::HtmlDjango { return vec![]; } let file_id = document.file_id(); - let Some(_line_index) = self.get_line_index(file_id) else { + let Some(_line_index) = self.get_line_index(uri) else { return vec![]; }; @@ -294,6 +471,11 @@ impl Store { } } +/// Check if a byte represents a word character (alphanumeric or underscore) +fn is_word_char(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || byte == b'_' +} + /// Apply text changes to content, handling multiple changes correctly fn apply_text_changes( content: &str, @@ -360,9 +542,10 @@ fn apply_text_changes( #[cfg(test)] mod tests { - use super::*; use tower_lsp_server::lsp_types::Range; + use super::*; + #[test] fn test_apply_single_character_insertion() { let content = "Hello world"; diff --git a/crates/djls-workspace/src/document/template.rs b/crates/djls-workspace/src/document/template.rs new file mode 100644 index 0000000..2a0547c --- /dev/null +++ b/crates/djls-workspace/src/document/template.rs @@ -0,0 +1,13 @@ +#[derive(Debug)] +pub enum ClosingBrace { + None, + PartialClose, // just } + FullClose, // %} +} + +#[derive(Debug)] +pub struct TemplateTagContext { + pub partial_tag: String, + pub closing_brace: ClosingBrace, + pub needs_leading_space: bool, +} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index a3db4c3..fb45cc1 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,14 +1,14 @@ mod bridge; mod db; +mod document; mod vfs; -mod watcher; -pub use bridge::FileStore; -pub use db::{ - parse_template, template_errors, Database, SourceFile, TemplateAst, TemplateLoaderOrder, -}; -pub use vfs::{FileKind, FileMeta, FileRecord, Revision, TextSource, Vfs, VfsSnapshot}; -pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; +pub use document::ClosingBrace; +pub use document::DocumentStore; +pub use document::LanguageId; +pub use document::LineIndex; +pub use document::TemplateTagContext; +pub use document::TextDocument; /// Stable, compact identifier for files across the subsystem. /// @@ -16,7 +16,7 @@ pub use watcher::{VfsWatcher, WatchConfig, WatchEvent}; /// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the /// lifetime of the VFS, even if the file's content or metadata changes. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub struct FileId(u32); +pub(crate) struct FileId(u32); impl FileId { /// Create a [`FileId`] from a raw u32 value. diff --git a/crates/djls-workspace/src/vfs.rs b/crates/djls-workspace/src/vfs/mod.rs similarity index 84% rename from crates/djls-workspace/src/vfs.rs rename to crates/djls-workspace/src/vfs/mod.rs index 08eb182..2b4b22f 100644 --- a/crates/djls-workspace/src/vfs.rs +++ b/crates/djls-workspace/src/vfs/mod.rs @@ -4,25 +4,28 @@ //! and snapshotting. Downstream systems consume snapshots to avoid locking and to //! batch updates. -use anyhow::{anyhow, Result}; +mod watcher; + +use std::collections::hash_map::DefaultHasher; +use std::collections::HashMap; +use std::fs; +use std::hash::Hash; +use std::hash::Hasher; +use std::sync::atomic::AtomicU32; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use anyhow::anyhow; +use anyhow::Result; use camino::Utf8PathBuf; use dashmap::DashMap; -use std::collections::hash_map::DefaultHasher; -use std::fs; -use std::hash::{Hash, Hasher}; -use std::{ - collections::HashMap, - sync::{ - atomic::{AtomicU32, AtomicU64, Ordering}, - Arc, - }, -}; use url::Url; +use watcher::VfsWatcher; +use watcher::WatchConfig; +use watcher::WatchEvent; -use super::{ - watcher::{VfsWatcher, WatchConfig, WatchEvent}, - FileId, -}; +use super::FileId; /// Monotonic counter representing global VFS state. /// @@ -30,18 +33,18 @@ use super::{ /// This provides a cheap way to detect if any changes have occurred since /// a previous snapshot was taken. #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default, PartialOrd, Ord)] -pub struct Revision(u64); +pub(crate) struct Revision(u64); impl Revision { /// Create a [`Revision`] from a raw u64 value. #[must_use] - pub fn from_raw(raw: u64) -> Self { + fn from_raw(raw: u64) -> Self { Revision(raw) } /// Get the underlying u64 value. #[must_use] - pub fn value(self) -> u64 { + fn value(self) -> u64 { self.0 } } @@ -65,11 +68,11 @@ pub enum FileKind { /// [`FileMeta`] contains all non-content information about a file, including its /// identity (URI), filesystem path, and classification. #[derive(Clone, Debug)] -pub struct FileMeta { +pub(crate) struct FileMeta { /// The file's URI (typically file:// scheme) - pub uri: Url, + uri: Url, /// The file's path in the filesystem - pub path: Utf8PathBuf, + path: Utf8PathBuf, /// Classification for routing to analyzers pub kind: FileKind, } @@ -80,7 +83,7 @@ pub struct FileMeta { /// debugging and understanding the current state of the VFS. All variants hold /// `Arc` for efficient sharing. #[derive(Clone)] -pub enum TextSource { +pub(crate) enum TextSource { /// Content loaded from disk Disk(Arc), /// Content from LSP client overlay (in-memory edits) @@ -89,18 +92,47 @@ pub enum TextSource { Generated(Arc), } +/// Content hash for efficient change detection. +/// +/// [`FileHash`] encapsulates the hashing logic used to detect when file content +/// has changed, avoiding unnecessary recomputation in downstream systems like Salsa. +#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] +struct FileHash(u64); + +impl FileHash { + /// Compute hash from text source content. + fn from_text_source(src: &TextSource) -> Self { + let s: &str = match src { + TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, + }; + let mut h = DefaultHasher::new(); + s.hash(&mut h); + Self(h.finish()) + } + + /// Check if this hash differs from another, indicating content changed. + fn differs_from(self, other: Self) -> bool { + self.0 != other.0 + } + + /// Get raw hash value (for debugging/logging). + fn raw(self) -> u64 { + self.0 + } +} + /// Complete record of a file in the VFS. /// /// [`FileRecord`] combines metadata, current text content, and a content hash /// for efficient change detection. #[derive(Clone)] -pub struct FileRecord { +pub(crate) struct FileRecord { /// File metadata (URI, path, kind, version) pub meta: FileMeta, /// Current text content and its source - pub text: TextSource, + text: TextSource, /// Hash of current content for change detection - pub hash: u64, + hash: FileHash, } /// Thread-safe virtual file system with change tracking. @@ -129,7 +161,7 @@ impl Vfs { /// Returns the existing [`FileId`] if the URI is already known, or creates a new /// [`FileRecord`] with the provided metadata and text. This method computes and /// stores a content hash for change detection. - pub fn intern_file( + pub(crate) fn intern_file( &self, uri: Url, path: Utf8PathBuf, @@ -145,7 +177,7 @@ impl Vfs { path: path.clone(), kind, }; - let hash = content_hash(&text); + let hash = FileHash::from_text_source(&text); self.by_uri.insert(uri, id); self.by_path.insert(path, id); self.files.insert(id, FileRecord { meta, text, hash }); @@ -159,14 +191,14 @@ impl Vfs { /// (detected via hash comparison). /// /// Returns a tuple of (new global revision, whether content changed). - pub fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { + pub(crate) fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { let mut rec = self .files .get_mut(&id) .ok_or_else(|| anyhow!("unknown file: {:?}", id))?; let next = TextSource::Overlay(new_text); - let new_hash = content_hash(&next); - let changed = new_hash != rec.hash; + let new_hash = FileHash::from_text_source(&next); + let changed = new_hash.differs_from(rec.hash); if changed { rec.text = next; rec.hash = new_hash; @@ -183,7 +215,7 @@ impl Vfs { /// Materializes a consistent view of all files for downstream consumers. /// The snapshot includes the current revision and a clone of all file records. /// This operation is relatively cheap due to `Arc` sharing of text content. - pub fn snapshot(&self) -> VfsSnapshot { + pub(crate) fn snapshot(&self) -> VfsSnapshot { VfsSnapshot { revision: Revision::from_raw(self.head.load(Ordering::SeqCst)), files: self @@ -268,11 +300,11 @@ impl Vfs { .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; let new_text = TextSource::Disk(Arc::from(content.as_str())); - let new_hash = content_hash(&new_text); + let new_hash = FileHash::from_text_source(&new_text); // Update the file if content changed if let Some(mut record) = self.files.get_mut(&file_id) { - if record.hash != new_hash { + if new_hash.differs_from(record.hash) { record.text = new_text; record.hash = new_hash; self.head.fetch_add(1, Ordering::SeqCst); @@ -301,28 +333,15 @@ impl Default for Vfs { } } -/// Compute a stable hash over file content. -/// -/// Used for efficient change detection - if the hash hasn't changed, -/// the content hasn't changed, avoiding unnecessary Salsa invalidations. -fn content_hash(src: &TextSource) -> u64 { - let s: &str = match src { - TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, - }; - let mut h = DefaultHasher::new(); - s.hash(&mut h); - h.finish() -} - /// Immutable snapshot view of the VFS at a specific revision. /// /// [`VfsSnapshot`] provides a consistent view of all files for downstream consumers, /// avoiding the need for locking during processing. Snapshots are created atomically /// and can be safely shared across threads. #[derive(Clone)] -pub struct VfsSnapshot { +pub(crate) struct VfsSnapshot { /// The global revision at the time of snapshot - pub revision: Revision, + revision: Revision, /// All files in the VFS at snapshot time pub files: HashMap, } diff --git a/crates/djls-workspace/src/watcher.rs b/crates/djls-workspace/src/vfs/watcher.rs similarity index 97% rename from crates/djls-workspace/src/watcher.rs rename to crates/djls-workspace/src/vfs/watcher.rs index 55fa9a7..aee0467 100644 --- a/crates/djls-workspace/src/watcher.rs +++ b/crates/djls-workspace/src/vfs/watcher.rs @@ -4,15 +4,21 @@ //! and synchronize them with the VFS. It uses cross-platform file watching with //! debouncing to handle rapid changes efficiently. -use anyhow::{anyhow, Result}; +use std::collections::HashMap; +use std::sync::mpsc; +use std::thread; +use std::time::Duration; +use std::time::Instant; + +use anyhow::anyhow; +use anyhow::Result; use camino::Utf8PathBuf; -use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; -use std::{ - collections::HashMap, - sync::mpsc, - thread, - time::{Duration, Instant}, -}; +use notify::Config; +use notify::Event; +use notify::EventKind; +use notify::RecommendedWatcher; +use notify::RecursiveMode; +use notify::Watcher; /// Event types that can occur in the file system. /// From 3bf25ac2040fdc52993dad90f02f59aadfe575a2 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 13:11:13 -0500 Subject: [PATCH 10/30] must use! --- .../djls-workspace/src/document/line_index.rs | 3 + crates/djls-workspace/src/document/store.rs | 119 +++--------------- 2 files changed, 23 insertions(+), 99 deletions(-) diff --git a/crates/djls-workspace/src/document/line_index.rs b/crates/djls-workspace/src/document/line_index.rs index 39f1fde..cbbf645 100644 --- a/crates/djls-workspace/src/document/line_index.rs +++ b/crates/djls-workspace/src/document/line_index.rs @@ -9,6 +9,7 @@ pub struct LineIndex { } impl LineIndex { + #[must_use] pub fn new(text: &str) -> Self { let mut line_starts = vec![0]; let mut line_starts_utf16 = vec![0]; @@ -32,6 +33,7 @@ impl LineIndex { } } + #[must_use] pub fn offset(&self, position: Position) -> Option { let line_start = self.line_starts.get(position.line as usize)?; @@ -73,6 +75,7 @@ impl LineIndex { } #[allow(dead_code)] + #[must_use] pub fn position(&self, offset: u32) -> Position { let line = match self.line_starts.binary_search(&offset) { Ok(line) => line, diff --git a/crates/djls-workspace/src/document/store.rs b/crates/djls-workspace/src/document/store.rs index 8284107..3f8e77e 100644 --- a/crates/djls-workspace/src/document/store.rs +++ b/crates/djls-workspace/src/document/store.rs @@ -49,6 +49,7 @@ impl Default for DocumentStore { } impl DocumentStore { + #[must_use] pub fn new() -> Self { Self::default() } @@ -134,88 +135,6 @@ impl DocumentStore { Ok(()) } - pub fn handle_did_open(&mut self, params: &DidOpenTextDocumentParams) -> Result<()> { - let uri_str = params.text_document.uri.to_string(); - let uri = params.text_document.uri.clone(); - let version = params.text_document.version; - let content = params.text_document.text.clone(); - let language_id = LanguageId::from(params.text_document.language_id.as_str()); - let kind = FileKind::from(language_id.clone()); - - // Convert URI to Url for VFS - let vfs_url = - url::Url::parse(&uri.to_string()).map_err(|e| anyhow!("Invalid URI: {}", e))?; - - // Convert to path - simplified for now, just use URI string - let path = Utf8PathBuf::from(uri.as_str()); - - // Store content in VFS - let text_source = TextSource::Overlay(Arc::from(content.as_str())); - let file_id = self.vfs.intern_file(vfs_url, path, kind, text_source); - - // Set overlay content in VFS - self.vfs.set_overlay(file_id, Arc::from(content.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - // Create TextDocument metadata - let document = TextDocument::new( - uri_str.clone(), - version, - language_id.clone(), - file_id, - &content, - ); - self.documents.insert(uri_str, document); - - Ok(()) - } - - pub fn handle_did_change(&mut self, params: &DidChangeTextDocumentParams) -> Result<()> { - let uri_str = params.text_document.uri.as_str().to_string(); - let version = params.text_document.version; - - // Get document and file_id from the documents HashMap - let document = self - .documents - .get(&uri_str) - .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; - let file_id = document.file_id(); - - // Get current content from VFS - let snapshot = self.vfs.snapshot(); - let current_content = snapshot - .get_text(file_id) - .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - - // Get line index from the document (TextDocument now stores its own LineIndex) - let line_index = document.line_index(); - - // Apply text changes using the new function - let new_content = - apply_text_changes(¤t_content, ¶ms.content_changes, line_index)?; - - // Update TextDocument version and content - if let Some(document) = self.documents.get_mut(&uri_str) { - document.version = version; - document.update_content(&new_content); - } - - // Update VFS with new content - self.vfs - .set_overlay(file_id, Arc::from(new_content.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - Ok(()) - } - /// Close a document with the given URI. /// This removes the document from internal storage and cleans up resources. pub fn close_document(&mut self, uri: &url::Url) { @@ -228,31 +147,25 @@ impl DocumentStore { // The VFS will handle cleanup internally } - pub fn handle_did_close(&mut self, params: &DidCloseTextDocumentParams) { - let uri_str = params.text_document.uri.as_str(); - - // Remove TextDocument metadata - self.documents.remove(uri_str); - - // Note: We don't remove from VFS as it might be useful for caching - // The VFS will handle cleanup internally - } - + #[must_use] pub fn get_line_index(&self, uri: &str) -> Option<&LineIndex> { - self.documents.get(uri).map(|doc| doc.line_index()) + self.documents.get(uri).map(super::TextDocument::line_index) } #[allow(dead_code)] + #[must_use] pub fn get_version(&self, uri: &str) -> Option { - self.documents.get(uri).map(|doc| doc.version()) + self.documents.get(uri).map(super::TextDocument::version) } #[allow(dead_code)] + #[must_use] pub fn is_version_valid(&self, uri: &str, version: i32) -> bool { self.get_version(uri) == Some(version) } // TextDocument helper methods + #[must_use] pub fn get_document(&self, uri: &str) -> Option<&TextDocument> { self.documents.get(uri) } @@ -262,10 +175,12 @@ impl DocumentStore { } // URI-based query methods (new API) + #[must_use] pub fn get_document_by_url(&self, uri: &url::Url) -> Option<&TextDocument> { self.get_document(uri.as_str()) } + #[must_use] pub fn get_document_text(&self, uri: &url::Url) -> Option> { let document = self.get_document_by_url(uri)?; let file_id = document.file_id(); @@ -273,6 +188,7 @@ impl DocumentStore { snapshot.get_text(file_id) } + #[must_use] pub fn get_line_text(&self, uri: &url::Url, line: u32) -> Option { let document = self.get_document_by_url(uri)?; let snapshot = self.vfs.snapshot(); @@ -280,6 +196,7 @@ impl DocumentStore { document.get_line(content.as_ref(), line) } + #[must_use] pub fn get_word_at_position(&self, uri: &url::Url, position: Position) -> Option { // This is a simplified implementation - get the line and extract word at position let line_text = self.get_line_text(uri, position.line)?; @@ -312,11 +229,13 @@ impl DocumentStore { } // Position mapping methods + #[must_use] pub fn offset_to_position(&self, uri: &url::Url, offset: usize) -> Option { let document = self.get_document_by_url(uri)?; Some(document.offset_to_position(offset as u32)) } + #[must_use] pub fn position_to_offset(&self, uri: &url::Url, position: Position) -> Option { let document = self.get_document_by_url(uri)?; document @@ -325,6 +244,7 @@ impl DocumentStore { } // Template-specific methods + #[must_use] pub fn get_template_ast(&self, uri: &url::Url) -> Option> { let document = self.get_document_by_url(uri)?; let file_id = document.file_id(); @@ -332,10 +252,10 @@ impl DocumentStore { file_store.get_template_ast(file_id) } + #[must_use] pub fn get_template_errors(&self, uri: &url::Url) -> Vec { - let document = match self.get_document_by_url(uri) { - Some(doc) => doc, - None => return vec![], + let Some(document) = self.get_document_by_url(uri) else { + return vec![]; }; let file_id = document.file_id(); let file_store = self.file_store.lock().unwrap(); @@ -343,6 +263,7 @@ impl DocumentStore { errors.to_vec() } + #[must_use] pub fn get_template_context( &self, uri: &url::Url, @@ -354,6 +275,7 @@ impl DocumentStore { document.get_template_tag_context(content.as_ref(), position) } + #[must_use] pub fn get_completions( &self, uri: &str, @@ -379,8 +301,7 @@ impl DocumentStore { // Get template tag context from document let vfs_snapshot = self.vfs.snapshot(); let text_content = vfs_snapshot.get_text(file_id)?; - let content = text_content.as_ref(); - let context = document.get_template_tag_context(content, position)?; + let context = document.get_template_tag_context(text_content.as_ref(), position)?; let mut completions: Vec = tags .iter() From 269d4bceaeb2588ec1ae4c83ad61d9f82185f928 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 25 Aug 2025 13:19:52 -0500 Subject: [PATCH 11/30] lsp types --- crates/djls-server/src/client.rs | 47 ++++++------- crates/djls-server/src/logging.rs | 16 ++--- crates/djls-server/src/server.rs | 109 +++++++++++++++--------------- crates/djls-server/src/session.rs | 12 ++-- 4 files changed, 88 insertions(+), 96 deletions(-) diff --git a/crates/djls-server/src/client.rs b/crates/djls-server/src/client.rs index 35eb841..11a5f62 100644 --- a/crates/djls-server/src/client.rs +++ b/crates/djls-server/src/client.rs @@ -123,45 +123,38 @@ macro_rules! request { #[allow(dead_code)] pub mod messages { - use tower_lsp_server::lsp_types::MessageActionItem; - use tower_lsp_server::lsp_types::MessageType; - use tower_lsp_server::lsp_types::ShowDocumentParams; + use tower_lsp_server::lsp_types; use super::get_client; use super::Display; use super::Error; - notify!(log_message, message_type: MessageType, message: impl Display + Send + 'static); - notify!(show_message, message_type: MessageType, message: impl Display + Send + 'static); - request!(show_message_request, message_type: MessageType, message: impl Display + Send + 'static, actions: Option> ; Option); - request!(show_document, params: ShowDocumentParams ; bool); + notify!(log_message, message_type: lsp_types::MessageType, message: impl Display + Send + 'static); + notify!(show_message, message_type: lsp_types::MessageType, message: impl Display + Send + 'static); + request!(show_message_request, message_type: lsp_types::MessageType, message: impl Display + Send + 'static, actions: Option> ; Option); + request!(show_document, params: lsp_types::ShowDocumentParams ; bool); } #[allow(dead_code)] pub mod diagnostics { - use tower_lsp_server::lsp_types::Diagnostic; - use tower_lsp_server::lsp_types::Uri; + use tower_lsp_server::lsp_types; use super::get_client; - notify!(publish_diagnostics, uri: Uri, diagnostics: Vec, version: Option); + notify!(publish_diagnostics, uri: lsp_types::Uri, diagnostics: Vec, version: Option); notify_discard!(workspace_diagnostic_refresh,); } #[allow(dead_code)] pub mod workspace { - use tower_lsp_server::lsp_types::ApplyWorkspaceEditResponse; - use tower_lsp_server::lsp_types::ConfigurationItem; - use tower_lsp_server::lsp_types::LSPAny; - use tower_lsp_server::lsp_types::WorkspaceEdit; - use tower_lsp_server::lsp_types::WorkspaceFolder; + use tower_lsp_server::lsp_types; use super::get_client; use super::Error; - request!(apply_edit, edit: WorkspaceEdit ; ApplyWorkspaceEditResponse); - request!(configuration, items: Vec ; Vec); - request!(workspace_folders, ; Option>); + request!(apply_edit, edit: lsp_types::WorkspaceEdit ; lsp_types::ApplyWorkspaceEditResponse); + request!(configuration, items: Vec ; Vec); + request!(workspace_folders, ; Option>); } #[allow(dead_code)] @@ -176,19 +169,18 @@ pub mod editor { #[allow(dead_code)] pub mod capabilities { - use tower_lsp_server::lsp_types::Registration; - use tower_lsp_server::lsp_types::Unregistration; + use tower_lsp_server::lsp_types; use super::get_client; - notify_discard!(register_capability, registrations: Vec); - notify_discard!(unregister_capability, unregisterations: Vec); + notify_discard!(register_capability, registrations: Vec); + notify_discard!(unregister_capability, unregisterations: Vec); } #[allow(dead_code)] pub mod monitoring { use serde::Serialize; - use tower_lsp_server::lsp_types::ProgressToken; + use tower_lsp_server::lsp_types; use tower_lsp_server::Progress; use super::get_client; @@ -201,22 +193,21 @@ pub mod monitoring { } } - pub fn progress + Send>(token: ProgressToken, title: T) -> Option { + pub fn progress + Send>(token: lsp_types::ProgressToken, title: T) -> Option { get_client().map(|client| client.progress(token, title)) } } #[allow(dead_code)] pub mod protocol { - use tower_lsp_server::lsp_types::notification::Notification; - use tower_lsp_server::lsp_types::request::Request; + use tower_lsp_server::lsp_types; use super::get_client; use super::Error; pub fn send_notification(params: N::Params) where - N: Notification, + N: lsp_types::notification::Notification, N::Params: Send + 'static, { if let Some(client) = get_client() { @@ -228,7 +219,7 @@ pub mod protocol { pub async fn send_request(params: R::Params) -> Result where - R: Request, + R: lsp_types::request::Request, R::Params: Send + 'static, R::Result: Send + 'static, { diff --git a/crates/djls-server/src/logging.rs b/crates/djls-server/src/logging.rs index a540401..030af94 100644 --- a/crates/djls-server/src/logging.rs +++ b/crates/djls-server/src/logging.rs @@ -15,7 +15,7 @@ use std::sync::Arc; -use tower_lsp_server::lsp_types::MessageType; +use tower_lsp_server::lsp_types; use tracing::field::Visit; use tracing::Level; use tracing_appender::non_blocking::WorkerGuard; @@ -32,13 +32,13 @@ use tracing_subscriber::Registry; /// that are sent to the client. It filters events by level to avoid overwhelming /// the client with verbose trace logs. pub struct LspLayer { - send_message: Arc, + send_message: Arc, } impl LspLayer { pub fn new(send_message: F) -> Self where - F: Fn(MessageType, String) + Send + Sync + 'static, + F: Fn(lsp_types::MessageType, String) + Send + Sync + 'static, { Self { send_message: Arc::new(send_message), @@ -82,10 +82,10 @@ where let metadata = event.metadata(); let message_type = match *metadata.level() { - Level::ERROR => MessageType::ERROR, - Level::WARN => MessageType::WARNING, - Level::INFO => MessageType::INFO, - Level::DEBUG => MessageType::LOG, + Level::ERROR => lsp_types::MessageType::ERROR, + Level::WARN => lsp_types::MessageType::WARNING, + Level::INFO => lsp_types::MessageType::INFO, + Level::DEBUG => lsp_types::MessageType::LOG, Level::TRACE => { // Skip TRACE level - too verbose for LSP client // TODO: Add MessageType::Debug in LSP 3.18.0 @@ -112,7 +112,7 @@ where /// Returns a `WorkerGuard` that must be kept alive for the file logging to work. pub fn init_tracing(send_message: F) -> WorkerGuard where - F: Fn(MessageType, String) + Send + Sync + 'static, + F: Fn(lsp_types::MessageType, String) + Send + Sync + 'static, { let file_appender = tracing_appender::rolling::daily("/tmp", "djls.log"); let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index e61bb08..c5fa5d6 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -3,25 +3,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use tower_lsp_server::jsonrpc::Result as LspResult; -use tower_lsp_server::lsp_types::CompletionOptions; -use tower_lsp_server::lsp_types::CompletionParams; -use tower_lsp_server::lsp_types::CompletionResponse; -use tower_lsp_server::lsp_types::DidChangeConfigurationParams; -use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; -use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::InitializeResult; -use tower_lsp_server::lsp_types::InitializedParams; -use tower_lsp_server::lsp_types::OneOf; -use tower_lsp_server::lsp_types::SaveOptions; -use tower_lsp_server::lsp_types::ServerCapabilities; -use tower_lsp_server::lsp_types::ServerInfo; -use tower_lsp_server::lsp_types::TextDocumentSyncCapability; -use tower_lsp_server::lsp_types::TextDocumentSyncKind; -use tower_lsp_server::lsp_types::TextDocumentSyncOptions; -use tower_lsp_server::lsp_types::WorkspaceFoldersServerCapabilities; -use tower_lsp_server::lsp_types::WorkspaceServerCapabilities; +use tower_lsp_server::lsp_types; use tower_lsp_server::LanguageServer; use tracing_appender::non_blocking::WorkerGuard; @@ -91,7 +73,10 @@ impl DjangoLanguageServer { } impl LanguageServer for DjangoLanguageServer { - async fn initialize(&self, params: InitializeParams) -> LspResult { + async fn initialize( + &self, + params: lsp_types::InitializeParams, + ) -> LspResult { tracing::info!("Initializing server..."); let session = Session::new(¶ms); @@ -101,9 +86,9 @@ impl LanguageServer for DjangoLanguageServer { *session_lock = Some(session); } - Ok(InitializeResult { - capabilities: ServerCapabilities { - completion_provider: Some(CompletionOptions { + Ok(lsp_types::InitializeResult { + capabilities: lsp_types::ServerCapabilities { + completion_provider: Some(lsp_types::CompletionOptions { resolve_provider: Some(false), trigger_characters: Some(vec![ "{".to_string(), @@ -112,25 +97,25 @@ impl LanguageServer for DjangoLanguageServer { ]), ..Default::default() }), - workspace: Some(WorkspaceServerCapabilities { - workspace_folders: Some(WorkspaceFoldersServerCapabilities { + workspace: Some(lsp_types::WorkspaceServerCapabilities { + workspace_folders: Some(lsp_types::WorkspaceFoldersServerCapabilities { supported: Some(true), - change_notifications: Some(OneOf::Left(true)), + change_notifications: Some(lsp_types::OneOf::Left(true)), }), file_operations: None, }), - text_document_sync: Some(TextDocumentSyncCapability::Options( - TextDocumentSyncOptions { + text_document_sync: Some(lsp_types::TextDocumentSyncCapability::Options( + lsp_types::TextDocumentSyncOptions { open_close: Some(true), - change: Some(TextDocumentSyncKind::INCREMENTAL), + change: Some(lsp_types::TextDocumentSyncKind::INCREMENTAL), will_save: Some(false), will_save_wait_until: Some(false), - save: Some(SaveOptions::default().into()), + save: Some(lsp_types::SaveOptions::default().into()), }, )), ..Default::default() }, - server_info: Some(ServerInfo { + server_info: Some(lsp_types::ServerInfo { name: SERVER_NAME.to_string(), version: Some(SERVER_VERSION.to_string()), }), @@ -139,7 +124,7 @@ impl LanguageServer for DjangoLanguageServer { } #[allow(clippy::too_many_lines)] - async fn initialized(&self, _params: InitializedParams) { + async fn initialized(&self, _params: lsp_types::InitializedParams) { tracing::info!("Server received initialized notification."); self.with_session_task(|session_arc| async move { @@ -214,7 +199,7 @@ impl LanguageServer for DjangoLanguageServer { Ok(()) } - async fn did_open(&self, params: DidOpenTextDocumentParams) { + async fn did_open(&self, params: lsp_types::DidOpenTextDocumentParams) { tracing::info!("Opened document: {:?}", params.text_document.uri); self.with_session_mut(|session| { @@ -240,7 +225,7 @@ impl LanguageServer for DjangoLanguageServer { .await; } - async fn did_change(&self, params: DidChangeTextDocumentParams) { + async fn did_change(&self, params: lsp_types::DidChangeTextDocumentParams) { tracing::info!("Changed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { @@ -263,7 +248,7 @@ impl LanguageServer for DjangoLanguageServer { .await; } - async fn did_close(&self, params: DidCloseTextDocumentParams) { + async fn did_close(&self, params: lsp_types::DidCloseTextDocumentParams) { tracing::info!("Closed document: {:?}", params.text_document.uri); self.with_session_mut(|session| { @@ -279,7 +264,10 @@ impl LanguageServer for DjangoLanguageServer { .await; } - async fn completion(&self, params: CompletionParams) -> LspResult> { + async fn completion( + &self, + params: lsp_types::CompletionParams, + ) -> LspResult> { Ok(self .with_session(|session| { if let Some(project) = session.project() { @@ -289,33 +277,48 @@ impl LanguageServer for DjangoLanguageServer { // Convert LSP Uri to url::Url if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Some(context) = session.documents().get_template_context(&url, position) { + if let Some(context) = + session.documents().get_template_context(&url, position) + { // Use the context to generate completions - let mut completions: Vec = tags + let mut completions: Vec = tags .iter() .filter(|tag| { - context.partial_tag.is_empty() || tag.name().starts_with(&context.partial_tag) + context.partial_tag.is_empty() + || tag.name().starts_with(&context.partial_tag) }) .map(|tag| { - let leading_space = if context.needs_leading_space { " " } else { "" }; - tower_lsp_server::lsp_types::CompletionItem { + let leading_space = + if context.needs_leading_space { " " } else { "" }; + lsp_types::CompletionItem { label: tag.name().to_string(), - kind: Some(tower_lsp_server::lsp_types::CompletionItemKind::KEYWORD), - detail: Some(format!("Template tag from {}", tag.library())), + kind: Some(lsp_types::CompletionItemKind::KEYWORD), + detail: Some(format!( + "Template tag from {}", + tag.library() + )), documentation: tag.doc().as_ref().map(|doc| { - tower_lsp_server::lsp_types::Documentation::MarkupContent( - tower_lsp_server::lsp_types::MarkupContent { - kind: tower_lsp_server::lsp_types::MarkupKind::Markdown, + lsp_types::Documentation::MarkupContent( + lsp_types::MarkupContent { + kind: lsp_types::MarkupKind::Markdown, value: (*doc).to_string(), - } + }, ) }), insert_text: Some(match context.closing_brace { - djls_workspace::ClosingBrace::None => format!("{}{} %}}", leading_space, tag.name()), - djls_workspace::ClosingBrace::PartialClose => format!("{}{} %", leading_space, tag.name()), - djls_workspace::ClosingBrace::FullClose => format!("{}{} ", leading_space, tag.name()), + djls_workspace::ClosingBrace::None => { + format!("{}{} %}}", leading_space, tag.name()) + } + djls_workspace::ClosingBrace::PartialClose => { + format!("{}{} %", leading_space, tag.name()) + } + djls_workspace::ClosingBrace::FullClose => { + format!("{}{} ", leading_space, tag.name()) + } }), - insert_text_format: Some(tower_lsp_server::lsp_types::InsertTextFormat::PLAIN_TEXT), + insert_text_format: Some( + lsp_types::InsertTextFormat::PLAIN_TEXT, + ), ..Default::default() } }) @@ -325,7 +328,7 @@ impl LanguageServer for DjangoLanguageServer { None } else { completions.sort_by(|a, b| a.label.cmp(&b.label)); - Some(tower_lsp_server::lsp_types::CompletionResponse::Array(completions)) + Some(lsp_types::CompletionResponse::Array(completions)) } } else { None @@ -343,7 +346,7 @@ impl LanguageServer for DjangoLanguageServer { .await) } - async fn did_change_configuration(&self, _params: DidChangeConfigurationParams) { + async fn did_change_configuration(&self, _params: lsp_types::DidChangeConfigurationParams) { tracing::info!("Configuration change detected. Reloading settings..."); let project_path = self diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 8a584d3..1f6da45 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -4,9 +4,7 @@ use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::DocumentStore; use percent_encoding::percent_decode_str; -use tower_lsp_server::lsp_types::ClientCapabilities; -use tower_lsp_server::lsp_types::InitializeParams; -use tower_lsp_server::lsp_types::Uri; +use tower_lsp_server::lsp_types; #[derive(Default)] pub struct Session { @@ -15,14 +13,14 @@ pub struct Session { settings: Settings, #[allow(dead_code)] - client_capabilities: ClientCapabilities, + client_capabilities: lsp_types::ClientCapabilities, } impl Session { /// Determines the project root path from initialization parameters. /// /// Tries the current directory first, then falls back to the first workspace folder. - fn get_project_path(params: &InitializeParams) -> Option { + fn get_project_path(params: &lsp_types::InitializeParams) -> Option { // Try current directory first std::env::current_dir().ok().or_else(|| { // Fall back to the first workspace folder URI @@ -35,7 +33,7 @@ impl Session { } /// Converts a `file:` URI into an absolute `PathBuf`. - fn uri_to_pathbuf(uri: &Uri) -> Option { + fn uri_to_pathbuf(uri: &lsp_types::Uri) -> Option { // Check if the scheme is "file" if uri.scheme().is_none_or(|s| s.as_str() != "file") { return None; @@ -57,7 +55,7 @@ impl Session { Some(PathBuf::from(path_str)) } - pub fn new(params: &InitializeParams) -> Self { + pub fn new(params: &lsp_types::InitializeParams) -> Self { let project_path = Self::get_project_path(params); let (project, settings) = if let Some(path) = &project_path { From 4e3446f6ee2bf256f2e1fe42bfcfda0bccbc0e1d Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 27 Aug 2025 15:37:29 -0500 Subject: [PATCH 12/30] wip --- Cargo.lock | 103 +-- crates/djls-server/src/server.rs | 137 +--- crates/djls-server/src/session.rs | 83 ++- crates/djls-workspace/src/bridge.rs | 160 +---- crates/djls-workspace/src/db.rs | 94 ++- .../djls-workspace/src/document/language.rs | 2 +- crates/djls-workspace/src/document/mod.rs | 2 - crates/djls-workspace/src/document/store.rs | 643 ------------------ crates/djls-workspace/src/lib.rs | 32 +- crates/djls-workspace/src/lsp_system.rs | 154 +++++ crates/djls-workspace/src/system.rs | 118 ++++ crates/djls-workspace/src/test_db.rs | 25 + crates/djls-workspace/src/vfs/mod.rs | 367 ---------- crates/djls-workspace/src/vfs/watcher.rs | 325 --------- 14 files changed, 577 insertions(+), 1668 deletions(-) delete mode 100644 crates/djls-workspace/src/document/store.rs create mode 100644 crates/djls-workspace/src/lsp_system.rs create mode 100644 crates/djls-workspace/src/system.rs create mode 100644 crates/djls-workspace/src/test_db.rs delete mode 100644 crates/djls-workspace/src/vfs/mod.rs delete mode 100644 crates/djls-workspace/src/vfs/watcher.rs diff --git a/Cargo.lock b/Cargo.lock index d739a68..d017be4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -140,9 +140,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.2" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" +checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" dependencies = [ "serde", ] @@ -170,21 +170,21 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "camino" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d07aa9a93b00c76f71bc35d598bed923f6d4f3a9ca5c24b7737ae1a292841c0" +checksum = "dd0b03af37dad7a14518b7691d81acb0f8222604ad3d1b02f6b4bed5188c0cd5" [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "clap" -version = "4.5.45" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -192,9 +192,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.44" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -442,7 +442,7 @@ dependencies = [ "directories", "serde", "tempfile", - "thiserror 2.0.15", + "thiserror 2.0.16", "toml", ] @@ -498,7 +498,7 @@ dependencies = [ "insta", "serde", "tempfile", - "thiserror 2.0.15", + "thiserror 2.0.16", "toml", ] @@ -883,9 +883,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" dependencies = [ "equivalent", "hashbrown 0.15.5", @@ -903,7 +903,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "inotify-sys", "libc", ] @@ -940,11 +940,11 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "cfg-if", "libc", ] @@ -1010,7 +1010,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "libc", ] @@ -1106,7 +1106,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "fsevent-sys", "inotify", "kqueue", @@ -1224,9 +1224,9 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" @@ -1235,7 +1235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" dependencies = [ "memchr", - "thiserror 2.0.15", + "thiserror 2.0.16", "ucd-trie", ] @@ -1417,7 +1417,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", ] [[package]] @@ -1428,19 +1428,19 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.16", "libredox", - "thiserror 2.0.15", + "thiserror 2.0.16", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", + "regex-automata 0.4.10", + "regex-syntax 0.8.6", ] [[package]] @@ -1454,13 +1454,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax 0.8.6", ] [[package]] @@ -1471,9 +1471,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" [[package]] name = "ron" @@ -1482,7 +1482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64", - "bitflags 2.9.2", + "bitflags 2.9.3", "serde", "serde_derive", ] @@ -1516,7 +1516,7 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", "errno", "libc", "linux-raw-sys", @@ -1630,9 +1630,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.142" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -1765,15 +1765,15 @@ checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tempfile" -version = "3.20.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "15b61f8f20e3a6f7e0649d825294eaf317edce30f82cf6026e7e4cb9222a7d1e" dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1793,11 +1793,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.15", + "thiserror-impl 2.0.16", ] [[package]] @@ -1813,9 +1813,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", @@ -2129,13 +2129,14 @@ checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "url" -version = "2.5.4" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2393,9 +2394,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -2412,7 +2413,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.2", + "bitflags 2.9.3", ] [[package]] diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index c5fa5d6..e002658 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -202,25 +202,13 @@ impl LanguageServer for DjangoLanguageServer { async fn did_open(&self, params: lsp_types::DidOpenTextDocumentParams) { tracing::info!("Opened document: {:?}", params.text_document.uri); - self.with_session_mut(|session| { - let uri = params.text_document.uri.clone(); - let version = params.text_document.version; - let language_id = + self.with_session_mut(|_session| { + // TODO: Handle document open after refactoring + let _uri = params.text_document.uri.clone(); + let _version = params.text_document.version; + let _language_id = djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); - let text = params.text_document.text.clone(); - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Err(e) = - session - .documents_mut() - .open_document(url, version, language_id, text) - { - tracing::error!("Failed to handle did_open: {}", e); - } - } else { - tracing::error!("Invalid URI: {:?}", uri); - } + let _text = params.text_document.text.clone(); }) .await; } @@ -228,22 +216,11 @@ impl LanguageServer for DjangoLanguageServer { async fn did_change(&self, params: lsp_types::DidChangeTextDocumentParams) { tracing::info!("Changed document: {:?}", params.text_document.uri); - self.with_session_mut(|session| { - let uri = ¶ms.text_document.uri; - let version = params.text_document.version; - let changes = params.content_changes.clone(); - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Err(e) = session - .documents_mut() - .update_document(&url, version, changes) - { - tracing::error!("Failed to handle did_change: {}", e); - } - } else { - tracing::error!("Invalid URI: {:?}", uri); - } + self.with_session_mut(|_session| { + // TODO: Handle document change after refactoring + let _uri = ¶ms.text_document.uri; + let _version = params.text_document.version; + let _changes = params.content_changes.clone(); }) .await; } @@ -251,99 +228,19 @@ impl LanguageServer for DjangoLanguageServer { async fn did_close(&self, params: lsp_types::DidCloseTextDocumentParams) { tracing::info!("Closed document: {:?}", params.text_document.uri); - self.with_session_mut(|session| { - let uri = ¶ms.text_document.uri; - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - session.documents_mut().close_document(&url); - } else { - tracing::error!("Invalid URI: {:?}", uri); - } + self.with_session_mut(|_session| { + // TODO: Handle document close after refactoring + let _uri = ¶ms.text_document.uri; }) .await; } async fn completion( &self, - params: lsp_types::CompletionParams, + _params: lsp_types::CompletionParams, ) -> LspResult> { - Ok(self - .with_session(|session| { - if let Some(project) = session.project() { - if let Some(tags) = project.template_tags() { - let uri = ¶ms.text_document_position.text_document.uri; - let position = params.text_document_position.position; - - // Convert LSP Uri to url::Url - if let Ok(url) = url::Url::parse(&uri.to_string()) { - if let Some(context) = - session.documents().get_template_context(&url, position) - { - // Use the context to generate completions - let mut completions: Vec = tags - .iter() - .filter(|tag| { - context.partial_tag.is_empty() - || tag.name().starts_with(&context.partial_tag) - }) - .map(|tag| { - let leading_space = - if context.needs_leading_space { " " } else { "" }; - lsp_types::CompletionItem { - label: tag.name().to_string(), - kind: Some(lsp_types::CompletionItemKind::KEYWORD), - detail: Some(format!( - "Template tag from {}", - tag.library() - )), - documentation: tag.doc().as_ref().map(|doc| { - lsp_types::Documentation::MarkupContent( - lsp_types::MarkupContent { - kind: lsp_types::MarkupKind::Markdown, - value: (*doc).to_string(), - }, - ) - }), - insert_text: Some(match context.closing_brace { - djls_workspace::ClosingBrace::None => { - format!("{}{} %}}", leading_space, tag.name()) - } - djls_workspace::ClosingBrace::PartialClose => { - format!("{}{} %", leading_space, tag.name()) - } - djls_workspace::ClosingBrace::FullClose => { - format!("{}{} ", leading_space, tag.name()) - } - }), - insert_text_format: Some( - lsp_types::InsertTextFormat::PLAIN_TEXT, - ), - ..Default::default() - } - }) - .collect(); - - if completions.is_empty() { - None - } else { - completions.sort_by(|a, b| a.label.cmp(&b.label)); - Some(lsp_types::CompletionResponse::Array(completions)) - } - } else { - None - } - } else { - None - } - } else { - None - } - } else { - None - } - }) - .await) + // TODO: Handle completion after refactoring + Ok(None) } async fn did_change_configuration(&self, _params: lsp_types::DidChangeConfigurationParams) { diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 1f6da45..beae4ec 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,17 +1,59 @@ +use std::collections::HashMap; use std::path::PathBuf; +use std::sync::Arc; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::DocumentStore; +use djls_workspace::{FileSystem, StdFileSystem, db::Database}; use percent_encoding::percent_decode_str; +use salsa::StorageHandle; use tower_lsp_server::lsp_types; +use url::Url; -#[derive(Default)] pub struct Session { + /// The Django project configuration project: Option, - documents: DocumentStore, + + /// LSP server settings settings: Settings, + /// A thread-safe Salsa database handle that can be shared between threads. + /// + /// This implements the insight from [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) + /// where we're using the `StorageHandle` to create a thread-safe handle that can be + /// shared between threads. When we need to use it, we clone the handle to get a new reference. + /// + /// This handle allows us to create database instances as needed. + /// Even though we're using a single-threaded runtime, we still need + /// this to be thread-safe because of LSP trait requirements. + /// + /// Usage: + /// ```rust,ignore + /// // Clone the StorageHandle for use in an async context + /// let db_handle = session.db_handle.clone(); + /// + /// // Use it in an async context + /// async_fn(move || { + /// // Get a database from the handle + /// let storage = db_handle.into_storage(); + /// let db = Database::from_storage(storage); + /// + /// // Use the database + /// db.some_query(args) + /// }); + /// ``` + db_handle: StorageHandle, + + /// File system abstraction for reading files + file_system: Arc, + + /// Index of open documents with overlays (in-memory changes) + /// Maps document URL to its current content + overlays: HashMap, + + /// Tracks the session revision for change detection + revision: u64, + #[allow(dead_code)] client_capabilities: lsp_types::ClientCapabilities, } @@ -72,8 +114,11 @@ impl Session { Self { client_capabilities: params.capabilities.clone(), project, - documents: DocumentStore::new(), settings, + db_handle: StorageHandle::new(None), + file_system: Arc::new(StdFileSystem), + overlays: HashMap::new(), + revision: 0, } } @@ -85,13 +130,7 @@ impl Session { &mut self.project } - pub fn documents(&self) -> &DocumentStore { - &self.documents - } - pub fn documents_mut(&mut self) -> &mut DocumentStore { - &mut self.documents - } pub fn settings(&self) -> &Settings { &self.settings @@ -100,4 +139,28 @@ impl Session { pub fn set_settings(&mut self, settings: Settings) { self.settings = settings; } + + /// Get a database instance from the session. + /// + /// This creates a usable database from the handle, which can be used + /// to query and update data. The database itself is not Send/Sync, + /// but the StorageHandle is, allowing us to work with tower-lsp. + pub fn db(&self) -> Database { + let storage = self.db_handle.clone().into_storage(); + Database::from_storage(storage) + } +} + +impl Default for Session { + fn default() -> Self { + Self { + project: None, + settings: Settings::default(), + db_handle: StorageHandle::new(None), + file_system: Arc::new(StdFileSystem), + overlays: HashMap::new(), + revision: 0, + client_capabilities: lsp_types::ClientCapabilities::default(), + } + } } diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs index 0338262..2da695e 100644 --- a/crates/djls-workspace/src/bridge.rs +++ b/crates/djls-workspace/src/bridge.rs @@ -15,9 +15,8 @@ use super::db::Database; use super::db::SourceFile; use super::db::TemplateAst; use super::db::TemplateLoaderOrder; -use super::vfs::FileKind; -use super::vfs::VfsSnapshot; use super::FileId; +use super::FileKind; /// Owner of the Salsa [`Database`] plus the handles for updating inputs. /// @@ -39,7 +38,7 @@ impl FileStore { #[must_use] pub fn new() -> Self { Self { - db: Database::default(), + db: Database::new(), files: HashMap::new(), template_loader: None, } @@ -59,34 +58,26 @@ impl FileStore { } } - /// Mirror a VFS snapshot into Salsa inputs. - /// - /// This method is the core synchronization point between the VFS and Salsa. - /// It iterates through all files in the snapshot and: - /// - Creates [`SourceFile`] inputs for new files - /// - Updates `.text` and `.kind` only when changed to preserve incremental reuse - /// - /// The method is idempotent and minimizes Salsa invalidations by checking for - /// actual changes before updating inputs. - pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { - for (id, rec) in &snap.files { - let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); - let new_kind = rec.meta.kind; + // TODO: This will be replaced with direct file management + // pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { + // for (id, rec) in &snap.files { + // let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); + // let new_kind = rec.meta.kind; - if let Some(sf) = self.files.get(id) { - // Update if changed — avoid touching Salsa when not needed - if sf.kind(&self.db) != new_kind { - sf.set_kind(&mut self.db).to(new_kind); - } - if sf.text(&self.db).as_ref() != &*new_text { - sf.set_text(&mut self.db).to(new_text.clone()); - } - } else { - let sf = SourceFile::new(&self.db, new_kind, new_text); - self.files.insert(*id, sf); - } - } - } + // if let Some(sf) = self.files.get(id) { + // // Update if changed — avoid touching Salsa when not needed + // if sf.kind(&self.db) != new_kind { + // sf.set_kind(&mut self.db).to(new_kind); + // } + // if sf.text(&self.db).as_ref() != &*new_text { + // sf.set_text(&mut self.db).to(new_text.clone()); + // } + // } else { + // let sf = SourceFile::new(&self.db, new_kind, new_text); + // self.files.insert(*id, sf); + // } + // } + // } /// Get the text content of a file by its [`FileId`]. /// @@ -130,109 +121,6 @@ impl Default for FileStore { } } -#[cfg(test)] -mod tests { - use camino::Utf8PathBuf; - - use super::*; - use crate::vfs::TextSource; - use crate::vfs::Vfs; - - #[test] - fn test_filestore_template_ast_caching() { - let mut store = FileStore::new(); - let vfs = Vfs::default(); - - // Create a template file in VFS - let url = url::Url::parse("file:///test.html").unwrap(); - let path = Utf8PathBuf::from("/test.html"); - let content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); - let file_id = vfs.intern_file( - url.clone(), - path.clone(), - FileKind::Template, - TextSource::Overlay(content.clone()), - ); - vfs.set_overlay(file_id, content.clone()).unwrap(); - - // Apply VFS snapshot to FileStore - let snapshot = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot); - - // Get template AST - should parse and cache - let ast1 = store.get_template_ast(file_id); - assert!(ast1.is_some()); - - // Get again - should return cached - let ast2 = store.get_template_ast(file_id); - assert!(ast2.is_some()); - assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } - - #[test] - fn test_filestore_template_errors() { - let mut store = FileStore::new(); - let vfs = Vfs::default(); - - // Create a template with an unclosed tag - let url = url::Url::parse("file:///error.html").unwrap(); - let path = Utf8PathBuf::from("/error.html"); - let content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); // Missing closing - let file_id = vfs.intern_file( - url.clone(), - path.clone(), - FileKind::Template, - TextSource::Overlay(content.clone()), - ); - vfs.set_overlay(file_id, content).unwrap(); - - // Apply VFS snapshot - let snapshot = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot); - - // Get errors - should contain parsing errors - let errors = store.get_template_errors(file_id); - // The template has unclosed tags, so there should be errors - // We don't assert on specific error count as the parser may evolve - - // Verify errors are cached - let errors2 = store.get_template_errors(file_id); - assert!(Arc::ptr_eq(&errors, &errors2)); - } - - #[test] - fn test_filestore_invalidation_on_content_change() { - let mut store = FileStore::new(); - let vfs = Vfs::default(); - - // Create initial template - let url = url::Url::parse("file:///change.html").unwrap(); - let path = Utf8PathBuf::from("/change.html"); - let content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); - let file_id = vfs.intern_file( - url.clone(), - path.clone(), - FileKind::Template, - TextSource::Overlay(content1.clone()), - ); - vfs.set_overlay(file_id, content1).unwrap(); - - // Apply snapshot and get AST - let snapshot1 = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot1); - let ast1 = store.get_template_ast(file_id); - - // Change content - let content2: Arc = Arc::from("{% for item in items %}{{ item }}{% endfor %}"); - vfs.set_overlay(file_id, content2).unwrap(); - - // Apply new snapshot - let snapshot2 = vfs.snapshot(); - store.apply_vfs_snapshot(&snapshot2); - - // Get AST again - should be different due to content change - let ast2 = store.get_template_ast(file_id); - assert!(ast1.is_some() && ast2.is_some()); - assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } -} +// TODO: Re-enable tests after VFS removal is complete +// #[cfg(test)] +// mod tests { diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 2783eae..c542ea4 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -4,23 +4,36 @@ //! Inputs are kept minimal to avoid unnecessary recomputation. use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; #[cfg(test)] use std::sync::Mutex; -use djls_templates::Ast; +use dashmap::DashMap; +use url::Url; -use crate::vfs::FileKind; +use crate::{FileId, FileKind}; /// Salsa database root for workspace /// /// The [`Database`] provides default storage and, in tests, captures Salsa events for /// reuse/diagnostics. It serves as the core incremental computation engine, tracking /// dependencies and invalidations across all inputs and derived queries. +/// +/// This database also manages the file system overlay for the workspace, +/// mapping URLs to FileIds and storing file content. #[salsa::db] #[derive(Clone)] -#[cfg_attr(not(test), derive(Default))] pub struct Database { storage: salsa::Storage, + + /// Map from file URL to FileId (thread-safe) + files: DashMap, + + /// Map from FileId to file content (thread-safe) + content: DashMap>, + + /// Next FileId to allocate (thread-safe counter) + next_file_id: Arc, // The logs are only used for testing and demonstrating reuse: #[cfg(test)] @@ -45,11 +58,86 @@ impl Default for Database { } } }))), + files: DashMap::new(), + content: DashMap::new(), + next_file_id: Arc::new(AtomicU32::new(0)), logs, } } } +impl Database { + /// Create a new database instance + pub fn new() -> Self { + Self { + storage: salsa::Storage::new(None), + files: DashMap::new(), + content: DashMap::new(), + next_file_id: Arc::new(AtomicU32::new(0)), + #[cfg(test)] + logs: Arc::new(Mutex::new(None)), + } + } + + /// Create a new database instance from a storage handle. + /// This is used by Session::db() to create databases from the StorageHandle. + pub fn from_storage(storage: salsa::Storage) -> Self { + Self { + storage, + files: DashMap::new(), + content: DashMap::new(), + next_file_id: Arc::new(AtomicU32::new(0)), + #[cfg(test)] + logs: Arc::new(Mutex::new(None)), + } + } + + /// Add or update a file in the workspace + pub fn set_file(&mut self, url: Url, content: String, _kind: FileKind) { + let file_id = if let Some(existing_id) = self.files.get(&url) { + *existing_id + } else { + let new_id = FileId::from_raw(self.next_file_id.fetch_add(1, Ordering::SeqCst)); + self.files.insert(url.clone(), new_id); + new_id + }; + + let content = Arc::::from(content); + self.content.insert(file_id, content.clone()); + + // TODO: Update Salsa inputs here when we connect them + } + + /// Remove a file from the workspace + pub fn remove_file(&mut self, url: &Url) { + if let Some((_, file_id)) = self.files.remove(url) { + self.content.remove(&file_id); + // TODO: Remove from Salsa when we connect inputs + } + } + + /// Get the content of a file by URL + pub fn get_file_content(&self, url: &Url) -> Option> { + let file_id = self.files.get(url)?; + self.content.get(&*file_id).map(|content| content.clone()) + } + + /// Get the content of a file by FileId + pub(crate) fn get_content_by_id(&self, file_id: FileId) -> Option> { + self.content.get(&file_id).map(|content| content.clone()) + } + + /// Check if a file exists in the workspace + pub fn has_file(&self, url: &Url) -> bool { + self.files.contains_key(url) + } + + /// Get all file URLs in the workspace + pub fn files(&self) -> impl Iterator + use<'_> { + self.files.iter().map(|entry| entry.key().clone()) + } +} + #[salsa::db] impl salsa::Database for Database {} diff --git a/crates/djls-workspace/src/document/language.rs b/crates/djls-workspace/src/document/language.rs index 09f0bb5..65c322a 100644 --- a/crates/djls-workspace/src/document/language.rs +++ b/crates/djls-workspace/src/document/language.rs @@ -1,4 +1,4 @@ -use crate::vfs::FileKind; +use crate::FileKind; #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { diff --git a/crates/djls-workspace/src/document/mod.rs b/crates/djls-workspace/src/document/mod.rs index 840cd17..93d443f 100644 --- a/crates/djls-workspace/src/document/mod.rs +++ b/crates/djls-workspace/src/document/mod.rs @@ -1,11 +1,9 @@ mod language; mod line_index; -mod store; mod template; pub use language::LanguageId; pub use line_index::LineIndex; -pub use store::DocumentStore; pub use template::ClosingBrace; pub use template::TemplateTagContext; use tower_lsp_server::lsp_types::Position; diff --git a/crates/djls-workspace/src/document/store.rs b/crates/djls-workspace/src/document/store.rs deleted file mode 100644 index 3f8e77e..0000000 --- a/crates/djls-workspace/src/document/store.rs +++ /dev/null @@ -1,643 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::Mutex; - -use anyhow::anyhow; -use anyhow::Result; -use camino::Utf8PathBuf; -use djls_project::TemplateTags; -use tower_lsp_server::lsp_types::CompletionItem; -use tower_lsp_server::lsp_types::CompletionItemKind; -use tower_lsp_server::lsp_types::CompletionResponse; -use tower_lsp_server::lsp_types::Diagnostic; -use tower_lsp_server::lsp_types::DiagnosticSeverity; -use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; -use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; -use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; -use tower_lsp_server::lsp_types::Documentation; -use tower_lsp_server::lsp_types::InsertTextFormat; -use tower_lsp_server::lsp_types::MarkupContent; -use tower_lsp_server::lsp_types::MarkupKind; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; -use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; - -use crate::bridge::FileStore; -use crate::db::TemplateAst; -use crate::vfs::FileKind; -use crate::vfs::TextSource; -use crate::vfs::Vfs; -use crate::ClosingBrace; -use crate::LanguageId; -use crate::LineIndex; -use crate::TextDocument; - -pub struct DocumentStore { - vfs: Arc, - file_store: Arc>, - documents: HashMap, -} - -impl Default for DocumentStore { - fn default() -> Self { - Self { - vfs: Arc::new(Vfs::default()), - file_store: Arc::new(Mutex::new(FileStore::new())), - documents: HashMap::new(), - } - } -} - -impl DocumentStore { - #[must_use] - pub fn new() -> Self { - Self::default() - } - - /// Open a document with the given URI, version, language, and text content. - /// This creates a new TextDocument and stores it internally, hiding VFS details. - pub fn open_document( - &mut self, - uri: url::Url, - version: i32, - language_id: LanguageId, - text: String, - ) -> Result<()> { - let uri_str = uri.to_string(); - let kind = FileKind::from(language_id.clone()); - - // Convert URI to path - simplified for now, just use URI string - let path = Utf8PathBuf::from(uri.as_str()); - - // Store content in VFS - let text_source = TextSource::Overlay(Arc::from(text.as_str())); - let file_id = self.vfs.intern_file(uri, path, kind, text_source); - - // Set overlay content in VFS - self.vfs.set_overlay(file_id, Arc::from(text.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - // Create TextDocument with LineIndex - let document = TextDocument::new(uri_str.clone(), version, language_id, file_id, &text); - self.documents.insert(uri_str, document); - - Ok(()) - } - - /// Update a document with the given URI, version, and text changes. - /// This applies changes to the document and updates the VFS accordingly. - pub fn update_document( - &mut self, - uri: &url::Url, - version: i32, - changes: Vec, - ) -> Result<()> { - let uri_str = uri.to_string(); - - // Get document and file_id from the documents HashMap - let document = self - .documents - .get(&uri_str) - .ok_or_else(|| anyhow!("Document not found: {}", uri_str))?; - let file_id = document.file_id(); - - // Get current content from VFS - let snapshot = self.vfs.snapshot(); - let current_content = snapshot - .get_text(file_id) - .ok_or_else(|| anyhow!("File content not found: {}", uri_str))?; - - // Get line index from the document - let line_index = document.line_index(); - - // Apply text changes using the existing function - let new_content = apply_text_changes(¤t_content, &changes, line_index)?; - - // Update TextDocument version and content - if let Some(document) = self.documents.get_mut(&uri_str) { - document.version = version; - document.update_content(&new_content); - } - - // Update VFS with new content - self.vfs - .set_overlay(file_id, Arc::from(new_content.as_str()))?; - - // Sync VFS snapshot to FileStore for Salsa tracking - let snapshot = self.vfs.snapshot(); - let mut file_store = self.file_store.lock().unwrap(); - file_store.apply_vfs_snapshot(&snapshot); - - Ok(()) - } - - /// Close a document with the given URI. - /// This removes the document from internal storage and cleans up resources. - pub fn close_document(&mut self, uri: &url::Url) { - let uri_str = uri.as_str(); - - // Remove TextDocument metadata - self.documents.remove(uri_str); - - // Note: We don't remove from VFS as it might be useful for caching - // The VFS will handle cleanup internally - } - - #[must_use] - pub fn get_line_index(&self, uri: &str) -> Option<&LineIndex> { - self.documents.get(uri).map(super::TextDocument::line_index) - } - - #[allow(dead_code)] - #[must_use] - pub fn get_version(&self, uri: &str) -> Option { - self.documents.get(uri).map(super::TextDocument::version) - } - - #[allow(dead_code)] - #[must_use] - pub fn is_version_valid(&self, uri: &str, version: i32) -> bool { - self.get_version(uri) == Some(version) - } - - // TextDocument helper methods - #[must_use] - pub fn get_document(&self, uri: &str) -> Option<&TextDocument> { - self.documents.get(uri) - } - - pub fn get_document_mut(&mut self, uri: &str) -> Option<&mut TextDocument> { - self.documents.get_mut(uri) - } - - // URI-based query methods (new API) - #[must_use] - pub fn get_document_by_url(&self, uri: &url::Url) -> Option<&TextDocument> { - self.get_document(uri.as_str()) - } - - #[must_use] - pub fn get_document_text(&self, uri: &url::Url) -> Option> { - let document = self.get_document_by_url(uri)?; - let file_id = document.file_id(); - let snapshot = self.vfs.snapshot(); - snapshot.get_text(file_id) - } - - #[must_use] - pub fn get_line_text(&self, uri: &url::Url, line: u32) -> Option { - let document = self.get_document_by_url(uri)?; - let snapshot = self.vfs.snapshot(); - let content = snapshot.get_text(document.file_id())?; - document.get_line(content.as_ref(), line) - } - - #[must_use] - pub fn get_word_at_position(&self, uri: &url::Url, position: Position) -> Option { - // This is a simplified implementation - get the line and extract word at position - let line_text = self.get_line_text(uri, position.line)?; - let char_pos: usize = position.character.try_into().ok()?; - - if char_pos >= line_text.len() { - return None; - } - - // Find word boundaries (simplified - considers alphanumeric and underscore as word chars) - let line_bytes = line_text.as_bytes(); - let mut start = char_pos; - let mut end = char_pos; - - // Find start of word - while start > 0 && is_word_char(line_bytes[start - 1]) { - start -= 1; - } - - // Find end of word - while end < line_text.len() && is_word_char(line_bytes[end]) { - end += 1; - } - - if start < end { - Some(line_text[start..end].to_string()) - } else { - None - } - } - - // Position mapping methods - #[must_use] - pub fn offset_to_position(&self, uri: &url::Url, offset: usize) -> Option { - let document = self.get_document_by_url(uri)?; - Some(document.offset_to_position(offset as u32)) - } - - #[must_use] - pub fn position_to_offset(&self, uri: &url::Url, position: Position) -> Option { - let document = self.get_document_by_url(uri)?; - document - .position_to_offset(position) - .map(|offset| offset as usize) - } - - // Template-specific methods - #[must_use] - pub fn get_template_ast(&self, uri: &url::Url) -> Option> { - let document = self.get_document_by_url(uri)?; - let file_id = document.file_id(); - let file_store = self.file_store.lock().unwrap(); - file_store.get_template_ast(file_id) - } - - #[must_use] - pub fn get_template_errors(&self, uri: &url::Url) -> Vec { - let Some(document) = self.get_document_by_url(uri) else { - return vec![]; - }; - let file_id = document.file_id(); - let file_store = self.file_store.lock().unwrap(); - let errors = file_store.get_template_errors(file_id); - errors.to_vec() - } - - #[must_use] - pub fn get_template_context( - &self, - uri: &url::Url, - position: Position, - ) -> Option { - let document = self.get_document_by_url(uri)?; - let snapshot = self.vfs.snapshot(); - let content = snapshot.get_text(document.file_id())?; - document.get_template_tag_context(content.as_ref(), position) - } - - #[must_use] - pub fn get_completions( - &self, - uri: &str, - position: Position, - tags: &TemplateTags, - ) -> Option { - // Check if this is a Django template using TextDocument metadata - let document = self.get_document(uri)?; - if document.language_id() != LanguageId::HtmlDjango { - return None; - } - - // Try to get cached AST from FileStore for better context analysis - // This demonstrates using the cached AST, though we still fall back to string parsing - let file_id = document.file_id(); - let file_store = self.file_store.lock().unwrap(); - if let Some(_ast) = file_store.get_template_ast(file_id) { - // TODO: In a future enhancement, we could use the AST to provide - // more intelligent completions based on the current node context - // For now, we continue with the existing string-based approach - } - - // Get template tag context from document - let vfs_snapshot = self.vfs.snapshot(); - let text_content = vfs_snapshot.get_text(file_id)?; - let context = document.get_template_tag_context(text_content.as_ref(), position)?; - - let mut completions: Vec = tags - .iter() - .filter(|tag| { - context.partial_tag.is_empty() || tag.name().starts_with(&context.partial_tag) - }) - .map(|tag| { - let leading_space = if context.needs_leading_space { " " } else { "" }; - CompletionItem { - label: tag.name().to_string(), - kind: Some(CompletionItemKind::KEYWORD), - detail: Some(format!("Template tag from {}", tag.library())), - documentation: tag.doc().as_ref().map(|doc| { - Documentation::MarkupContent(MarkupContent { - kind: MarkupKind::Markdown, - value: (*doc).to_string(), - }) - }), - insert_text: Some(match context.closing_brace { - ClosingBrace::None => format!("{}{} %}}", leading_space, tag.name()), - ClosingBrace::PartialClose => format!("{}{} %", leading_space, tag.name()), - ClosingBrace::FullClose => format!("{}{} ", leading_space, tag.name()), - }), - insert_text_format: Some(InsertTextFormat::PLAIN_TEXT), - ..Default::default() - } - }) - .collect(); - - if completions.is_empty() { - None - } else { - completions.sort_by(|a, b| a.label.cmp(&b.label)); - Some(CompletionResponse::Array(completions)) - } - } - - /// Get template parsing diagnostics for a file. - /// - /// This method uses the cached template errors from Salsa to generate LSP diagnostics. - /// The errors are only re-computed when the file content changes, providing efficient - /// incremental error reporting. - pub fn get_template_diagnostics(&self, uri: &str) -> Vec { - let Some(document) = self.get_document(uri) else { - return vec![]; - }; - - // Only process template files - if document.language_id() != LanguageId::HtmlDjango { - return vec![]; - } - - let file_id = document.file_id(); - let Some(_line_index) = self.get_line_index(uri) else { - return vec![]; - }; - - // Get cached template errors from FileStore - let file_store = self.file_store.lock().unwrap(); - let errors = file_store.get_template_errors(file_id); - - // Convert template errors to LSP diagnostics - errors - .iter() - .map(|error| { - // For now, we'll place all errors at the start of the file - // In a future enhancement, we could use error spans for precise locations - let range = Range { - start: Position { - line: 0, - character: 0, - }, - end: Position { - line: 0, - character: 0, - }, - }; - - Diagnostic { - range, - severity: Some(DiagnosticSeverity::ERROR), - source: Some("djls-templates".to_string()), - message: error.clone(), - ..Default::default() - } - }) - .collect() - } -} - -/// Check if a byte represents a word character (alphanumeric or underscore) -fn is_word_char(byte: u8) -> bool { - byte.is_ascii_alphanumeric() || byte == b'_' -} - -/// Apply text changes to content, handling multiple changes correctly -fn apply_text_changes( - content: &str, - changes: &[TextDocumentContentChangeEvent], - line_index: &LineIndex, -) -> Result { - if changes.is_empty() { - return Ok(content.to_string()); - } - - // Check for full document replacement first - for change in changes { - if change.range.is_none() { - return Ok(change.text.clone()); - } - } - - // Sort changes by start position in reverse order (end to start) - let mut sorted_changes = changes.to_vec(); - sorted_changes.sort_by(|a, b| { - match (a.range, b.range) { - (Some(range_a), Some(range_b)) => { - // Primary sort: by line (reverse) - let line_cmp = range_b.start.line.cmp(&range_a.start.line); - if line_cmp == std::cmp::Ordering::Equal { - // Secondary sort: by character (reverse) - range_b.start.character.cmp(&range_a.start.character) - } else { - line_cmp - } - } - _ => std::cmp::Ordering::Equal, - } - }); - - let mut result = content.to_string(); - - for change in &sorted_changes { - if let Some(range) = change.range { - // Convert UTF-16 positions to UTF-8 offsets - let start_offset = line_index - .offset_utf16(range.start, &result) - .ok_or_else(|| anyhow!("Invalid start position: {:?}", range.start))?; - let end_offset = line_index - .offset_utf16(range.end, &result) - .ok_or_else(|| anyhow!("Invalid end position: {:?}", range.end))?; - - if start_offset as usize > result.len() || end_offset as usize > result.len() { - return Err(anyhow!( - "Offset out of bounds: start={}, end={}, len={}", - start_offset, - end_offset, - result.len() - )); - } - - // Apply the change - result.replace_range(start_offset as usize..end_offset as usize, &change.text); - } - } - - Ok(result) -} - -#[cfg(test)] -mod tests { - use tower_lsp_server::lsp_types::Range; - - use super::*; - - #[test] - fn test_apply_single_character_insertion() { - let content = "Hello world"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 6), Position::new(0, 6))), - range_length: None, - text: "beautiful ".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "Hello beautiful world"); - } - - #[test] - fn test_apply_single_character_deletion() { - let content = "Hello world"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 5), Position::new(0, 6))), - range_length: None, - text: String::new(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "Helloworld"); - } - - #[test] - fn test_apply_multiple_changes_in_reverse_order() { - let content = "line 1\nline 2\nline 3"; - let line_index = LineIndex::new(content); - - // Insert "new " at position (1, 0) and "another " at position (0, 0) - let changes = vec![ - TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), - range_length: None, - text: "another ".to_string(), - }, - TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(1, 0), Position::new(1, 0))), - range_length: None, - text: "new ".to_string(), - }, - ]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "another line 1\nnew line 2\nline 3"); - } - - #[test] - fn test_apply_multiline_replacement() { - let content = "line 1\nline 2\nline 3"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(2, 6))), - range_length: None, - text: "completely new content".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "completely new content"); - } - - #[test] - fn test_apply_full_document_replacement() { - let content = "old content"; - let line_index = LineIndex::new(content); - - let changes = vec![TextDocumentContentChangeEvent { - range: None, - range_length: None, - text: "brand new content".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "brand new content"); - } - - #[test] - fn test_utf16_line_index_basic() { - let content = "hello world"; - let line_index = LineIndex::new(content); - - // ASCII characters should have 1:1 UTF-8:UTF-16 mapping - let pos = Position::new(0, 6); - let offset = line_index.offset_utf16(pos, content).unwrap(); - assert_eq!(offset, 6); - assert_eq!(&content[6..7], "w"); - } - - #[test] - fn test_utf16_line_index_with_emoji() { - let content = "hello 👋 world"; - let line_index = LineIndex::new(content); - - // 👋 is 2 UTF-16 code units but 4 UTF-8 bytes - let pos_after_emoji = Position::new(0, 8); // UTF-16 position after "hello 👋" - let offset = line_index.offset_utf16(pos_after_emoji, content).unwrap(); - - // Should point to the space before "world" - assert_eq!(offset, 10); // UTF-8 byte offset - assert_eq!(&content[10..11], " "); - } - - #[test] - fn test_utf16_line_index_multiline() { - let content = "first line\nsecond line"; - let line_index = LineIndex::new(content); - - let pos = Position::new(1, 7); // Position at 'l' in "line" on second line - let offset = line_index.offset_utf16(pos, content).unwrap(); - assert_eq!(offset, 18); // 11 (first line + \n) + 7 - assert_eq!(&content[18..19], "l"); - } - - #[test] - fn test_apply_changes_with_emoji() { - let content = "hello 👋 world"; - let line_index = LineIndex::new(content); - - // Insert text after the space following the emoji (UTF-16 position 9) - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 9), Position::new(0, 9))), - range_length: None, - text: "beautiful ".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "hello 👋 beautiful world"); - } - - #[test] - fn test_line_index_utf16_tracking() { - let content = "a👋b"; - let line_index = LineIndex::new(content); - - // Check UTF-16 line starts are tracked correctly - assert_eq!(line_index.line_starts_utf16, vec![0]); - assert_eq!(line_index.length_utf16, 4); // 'a' (1) + 👋 (2) + 'b' (1) = 4 UTF-16 units - assert_eq!(line_index.length, 6); // 'a' (1) + 👋 (4) + 'b' (1) = 6 UTF-8 bytes - } - - #[test] - fn test_edge_case_changes_at_boundaries() { - let content = "abc"; - let line_index = LineIndex::new(content); - - // Insert at beginning - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 0), Position::new(0, 0))), - range_length: None, - text: "start".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "startabc"); - - // Insert at end - let line_index = LineIndex::new(content); - let changes = vec![TextDocumentContentChangeEvent { - range: Some(Range::new(Position::new(0, 3), Position::new(0, 3))), - range_length: None, - text: "end".to_string(), - }]; - - let result = apply_text_changes(content, &changes, &line_index).unwrap(); - assert_eq!(result, "abcend"); - } -} diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index fb45cc1..9fbb34f 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,22 +1,33 @@ mod bridge; -mod db; +pub mod db; mod document; -mod vfs; +mod lsp_system; +mod system; -pub use document::ClosingBrace; -pub use document::DocumentStore; -pub use document::LanguageId; -pub use document::LineIndex; -pub use document::TemplateTagContext; -pub use document::TextDocument; +pub use db::Database; +pub use document::{TextDocument, LanguageId}; +pub use system::{FileSystem, StdFileSystem}; + +/// File classification for routing to analyzers. +/// +/// [`FileKind`] determines how a file should be processed by downstream analyzers. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKind { + /// Python source file + Python, + /// Django template file + Template, + /// Other file type + Other, +} /// Stable, compact identifier for files across the subsystem. /// /// [`FileId`] decouples file identity from paths/URIs, providing efficient keys for maps and /// Salsa inputs. Once assigned to a file (via its URI), a [`FileId`] remains stable for the -/// lifetime of the VFS, even if the file's content or metadata changes. +/// lifetime of the system. #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd)] -pub(crate) struct FileId(u32); +pub struct FileId(u32); impl FileId { /// Create a [`FileId`] from a raw u32 value. @@ -27,6 +38,7 @@ impl FileId { /// Get the underlying u32 index value. #[must_use] + #[allow(dead_code)] pub fn index(self) -> u32 { self.0 } diff --git a/crates/djls-workspace/src/lsp_system.rs b/crates/djls-workspace/src/lsp_system.rs new file mode 100644 index 0000000..b03c8e8 --- /dev/null +++ b/crates/djls-workspace/src/lsp_system.rs @@ -0,0 +1,154 @@ +//! LSP-aware file system wrapper that handles overlays +//! +//! This is the KEY pattern from Ruff - the LspSystem wraps a FileSystem +//! and intercepts reads to check for overlays first. This allows unsaved +//! changes to be used without going through Salsa. + +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use url::Url; + +use crate::system::FileSystem; + +/// LSP-aware file system that checks overlays before disk +/// +/// This is the critical piece that makes overlays work efficiently in Ruff's +/// architecture. Instead of updating Salsa for every keystroke, we intercept +/// file reads here and return overlay content when available. +pub struct LspSystem { + /// The underlying file system (usually StdFileSystem) + inner: Arc, + + /// Map of open document URLs to their overlay content + overlays: HashMap, +} + +impl LspSystem { + /// Create a new LspSystem wrapping the given file system + pub fn new(file_system: Arc) -> Self { + Self { + inner: file_system, + overlays: HashMap::new(), + } + } + + /// Set overlay content for a document + pub fn set_overlay(&mut self, url: Url, content: String) { + self.overlays.insert(url, content); + } + + /// Remove overlay content for a document + pub fn remove_overlay(&mut self, url: &Url) { + self.overlays.remove(url); + } + + /// Check if a document has an overlay + pub fn has_overlay(&self, url: &Url) -> bool { + self.overlays.contains_key(url) + } + + /// Get overlay content if it exists + pub fn get_overlay(&self, url: &Url) -> Option<&String> { + self.overlays.get(url) + } + + /// Convert a URL to a file path + fn url_to_path(url: &Url) -> Option { + if url.scheme() == "file" { + url.to_file_path().ok().or_else(|| { + // Fallback for simple conversion + Some(PathBuf::from(url.path())) + }) + } else { + None + } + } +} + +impl FileSystem for LspSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + // First check if we have an overlay for this path + // Convert path to URL for lookup + let url = Url::from_file_path(path) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid path"))?; + + if let Some(content) = self.overlays.get(&url) { + // Return overlay content instead of reading from disk + return Ok(content.clone()); + } + + // No overlay, read from underlying file system + self.inner.read_to_string(path) + } + + fn exists(&self, path: &Path) -> bool { + // Check overlays first + if let Ok(url) = Url::from_file_path(path) { + if self.overlays.contains_key(&url) { + return true; + } + } + + self.inner.exists(path) + } + + fn is_file(&self, path: &Path) -> bool { + // Overlays are always files + if let Ok(url) = Url::from_file_path(path) { + if self.overlays.contains_key(&url) { + return true; + } + } + + self.inner.is_file(path) + } + + fn is_directory(&self, path: &Path) -> bool { + // Overlays are never directories + if let Ok(url) = Url::from_file_path(path) { + if self.overlays.contains_key(&url) { + return false; + } + } + + self.inner.is_directory(path) + } + + fn read_directory(&self, path: &Path) -> io::Result> { + // Overlays don't affect directory listings + self.inner.read_directory(path) + } + + fn metadata(&self, path: &Path) -> io::Result { + // Can't provide metadata for overlays + self.inner.metadata(path) + } +} + +/// Extension trait for working with URL-based overlays +pub trait LspSystemExt { + /// Read file content by URL, checking overlays first + fn read_url(&self, url: &Url) -> io::Result; +} + +impl LspSystemExt for LspSystem { + fn read_url(&self, url: &Url) -> io::Result { + // Check overlays first + if let Some(content) = self.overlays.get(url) { + return Ok(content.clone()); + } + + // Convert URL to path and read from file system + if let Some(path_buf) = Self::url_to_path(url) { + self.inner.read_to_string(&path_buf) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("Cannot convert URL to path: {}", url), + )) + } + } +} \ No newline at end of file diff --git a/crates/djls-workspace/src/system.rs b/crates/djls-workspace/src/system.rs new file mode 100644 index 0000000..04a1b8a --- /dev/null +++ b/crates/djls-workspace/src/system.rs @@ -0,0 +1,118 @@ +//! File system abstraction following Ruff's pattern +//! +//! This module provides the FileSystem trait that abstracts file I/O operations. +//! This allows the LSP to work with both real files and in-memory overlays. + +use std::io; +use std::path::Path; + +/// Trait for file system operations +/// +/// This follows Ruff's pattern of abstracting file system operations behind a trait, +/// allowing different implementations for testing, in-memory operation, and real file access. +pub trait FileSystem: Send + Sync { + /// Read the entire contents of a file + fn read_to_string(&self, path: &Path) -> io::Result; + + /// Check if a path exists + fn exists(&self, path: &Path) -> bool; + + /// Check if a path is a file + fn is_file(&self, path: &Path) -> bool; + + /// Check if a path is a directory + fn is_directory(&self, path: &Path) -> bool; + + /// List directory contents + fn read_directory(&self, path: &Path) -> io::Result>; + + /// Get file metadata (size, modified time, etc.) + fn metadata(&self, path: &Path) -> io::Result; +} + +/// Standard file system implementation that uses std::fs +pub struct StdFileSystem; + +impl FileSystem for StdFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + std::fs::read_to_string(path) + } + + fn exists(&self, path: &Path) -> bool { + path.exists() + } + + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn is_directory(&self, path: &Path) -> bool { + path.is_dir() + } + + fn read_directory(&self, path: &Path) -> io::Result> { + let mut entries = Vec::new(); + for entry in std::fs::read_dir(path)? { + entries.push(entry?.path()); + } + Ok(entries) + } + + fn metadata(&self, path: &Path) -> io::Result { + std::fs::metadata(path) + } +} + +/// In-memory file system for testing +#[cfg(test)] +pub struct MemoryFileSystem { + files: std::collections::HashMap, +} + +#[cfg(test)] +impl MemoryFileSystem { + pub fn new() -> Self { + Self { + files: std::collections::HashMap::new(), + } + } + + pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { + self.files.insert(path, content); + } +} + +#[cfg(test)] +impl FileSystem for MemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + // Simplified for testing - no directories in memory filesystem + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + // Simplified for testing + Ok(Vec::new()) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Metadata not supported in memory filesystem", + )) + } +} \ No newline at end of file diff --git a/crates/djls-workspace/src/test_db.rs b/crates/djls-workspace/src/test_db.rs new file mode 100644 index 0000000..92683d5 --- /dev/null +++ b/crates/djls-workspace/src/test_db.rs @@ -0,0 +1,25 @@ +//! Test module to explore Salsa thread safety + +#[cfg(test)] +mod tests { + use crate::db::Database; + use std::thread; + + #[test] + fn test_database_clone() { + let db = Database::new(); + let _db2 = db.clone(); + println!("✅ Database can be cloned"); + } + + #[test] + #[ignore] // This will fail + fn test_database_send() { + let db = Database::new(); + let db2 = db.clone(); + + thread::spawn(move || { + let _ = db2; + }).join().unwrap(); + } +} diff --git a/crates/djls-workspace/src/vfs/mod.rs b/crates/djls-workspace/src/vfs/mod.rs deleted file mode 100644 index 2b4b22f..0000000 --- a/crates/djls-workspace/src/vfs/mod.rs +++ /dev/null @@ -1,367 +0,0 @@ -//! Change-tracked, concurrent virtual file system keyed by [`FileId`]. -//! -//! The VFS provides thread-safe, identity-stable storage with cheap change detection -//! and snapshotting. Downstream systems consume snapshots to avoid locking and to -//! batch updates. - -mod watcher; - -use std::collections::hash_map::DefaultHasher; -use std::collections::HashMap; -use std::fs; -use std::hash::Hash; -use std::hash::Hasher; -use std::sync::atomic::AtomicU32; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; -use std::sync::Arc; - -use anyhow::anyhow; -use anyhow::Result; -use camino::Utf8PathBuf; -use dashmap::DashMap; -use url::Url; -use watcher::VfsWatcher; -use watcher::WatchConfig; -use watcher::WatchEvent; - -use super::FileId; - -/// Monotonic counter representing global VFS state. -/// -/// [`Revision`] increments whenever file content changes occur in the VFS. -/// This provides a cheap way to detect if any changes have occurred since -/// a previous snapshot was taken. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default, PartialOrd, Ord)] -pub(crate) struct Revision(u64); - -impl Revision { - /// Create a [`Revision`] from a raw u64 value. - #[must_use] - fn from_raw(raw: u64) -> Self { - Revision(raw) - } - - /// Get the underlying u64 value. - #[must_use] - fn value(self) -> u64 { - self.0 - } -} - -/// File classification at the VFS layer. -/// -/// [`FileKind`] determines how a file should be processed by downstream analyzers. -/// This classification is performed when files are first ingested into the VFS. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub enum FileKind { - /// Python source file - Python, - /// Django template file - Template, - /// Other file type - Other, -} - -/// Metadata associated with a file in the VFS. -/// -/// [`FileMeta`] contains all non-content information about a file, including its -/// identity (URI), filesystem path, and classification. -#[derive(Clone, Debug)] -pub(crate) struct FileMeta { - /// The file's URI (typically file:// scheme) - uri: Url, - /// The file's path in the filesystem - path: Utf8PathBuf, - /// Classification for routing to analyzers - pub kind: FileKind, -} - -/// Source of text content in the VFS. -/// -/// [`TextSource`] tracks where file content originated from, which is useful for -/// debugging and understanding the current state of the VFS. All variants hold -/// `Arc` for efficient sharing. -#[derive(Clone)] -pub(crate) enum TextSource { - /// Content loaded from disk - Disk(Arc), - /// Content from LSP client overlay (in-memory edits) - Overlay(Arc), - /// Content generated programmatically - Generated(Arc), -} - -/// Content hash for efficient change detection. -/// -/// [`FileHash`] encapsulates the hashing logic used to detect when file content -/// has changed, avoiding unnecessary recomputation in downstream systems like Salsa. -#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] -struct FileHash(u64); - -impl FileHash { - /// Compute hash from text source content. - fn from_text_source(src: &TextSource) -> Self { - let s: &str = match src { - TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s, - }; - let mut h = DefaultHasher::new(); - s.hash(&mut h); - Self(h.finish()) - } - - /// Check if this hash differs from another, indicating content changed. - fn differs_from(self, other: Self) -> bool { - self.0 != other.0 - } - - /// Get raw hash value (for debugging/logging). - fn raw(self) -> u64 { - self.0 - } -} - -/// Complete record of a file in the VFS. -/// -/// [`FileRecord`] combines metadata, current text content, and a content hash -/// for efficient change detection. -#[derive(Clone)] -pub(crate) struct FileRecord { - /// File metadata (URI, path, kind, version) - pub meta: FileMeta, - /// Current text content and its source - text: TextSource, - /// Hash of current content for change detection - hash: FileHash, -} - -/// Thread-safe virtual file system with change tracking. -/// -/// [`Vfs`] provides concurrent access to file content with stable [`FileId`] assignment, -/// content hashing for change detection, and atomic snapshot generation. It uses -/// `DashMap` for lock-free concurrent access and atomic counters for revision tracking. -pub struct Vfs { - /// Atomic counter for generating unique [`FileId`]s - next_file_id: AtomicU32, - /// Map from URI to [`FileId`] for deduplication - by_uri: DashMap, - /// Map from [`FileId`] to [`FileRecord`] for content storage - files: DashMap, - /// Global revision counter, incremented on content changes - head: AtomicU64, - /// Optional file system watcher for external change detection - watcher: std::sync::Mutex>, - /// Map from filesystem path to [`FileId`] for watcher events - by_path: DashMap, -} - -impl Vfs { - /// Get or create a [`FileId`] for the given URI. - /// - /// Returns the existing [`FileId`] if the URI is already known, or creates a new - /// [`FileRecord`] with the provided metadata and text. This method computes and - /// stores a content hash for change detection. - pub(crate) fn intern_file( - &self, - uri: Url, - path: Utf8PathBuf, - kind: FileKind, - text: TextSource, - ) -> FileId { - if let Some(id) = self.by_uri.get(&uri).map(|entry| *entry) { - return id; - } - let id = FileId(self.next_file_id.fetch_add(1, Ordering::SeqCst)); - let meta = FileMeta { - uri: uri.clone(), - path: path.clone(), - kind, - }; - let hash = FileHash::from_text_source(&text); - self.by_uri.insert(uri, id); - self.by_path.insert(path, id); - self.files.insert(id, FileRecord { meta, text, hash }); - id - } - - /// Set overlay text for a file, typically from LSP didChange events. - /// - /// Updates the file's text to an Overlay variant with the new content. - /// Only increments the global revision if the content actually changed - /// (detected via hash comparison). - /// - /// Returns a tuple of (new global revision, whether content changed). - pub(crate) fn set_overlay(&self, id: FileId, new_text: Arc) -> Result<(Revision, bool)> { - let mut rec = self - .files - .get_mut(&id) - .ok_or_else(|| anyhow!("unknown file: {:?}", id))?; - let next = TextSource::Overlay(new_text); - let new_hash = FileHash::from_text_source(&next); - let changed = new_hash.differs_from(rec.hash); - if changed { - rec.text = next; - rec.hash = new_hash; - self.head.fetch_add(1, Ordering::SeqCst); - } - Ok(( - Revision::from_raw(self.head.load(Ordering::SeqCst)), - changed, - )) - } - - /// Create an immutable snapshot of the current VFS state. - /// - /// Materializes a consistent view of all files for downstream consumers. - /// The snapshot includes the current revision and a clone of all file records. - /// This operation is relatively cheap due to `Arc` sharing of text content. - pub(crate) fn snapshot(&self) -> VfsSnapshot { - VfsSnapshot { - revision: Revision::from_raw(self.head.load(Ordering::SeqCst)), - files: self - .files - .iter() - .map(|entry| (*entry.key(), entry.value().clone())) - .collect(), - } - } - - /// Enable file system watching with the given configuration. - /// - /// This starts monitoring the specified root directories for external changes. - /// Returns an error if file watching is disabled in the config or fails to start. - pub fn enable_file_watching(&self, config: WatchConfig) -> Result<()> { - let watcher = VfsWatcher::new(config)?; - *self - .watcher - .lock() - .map_err(|e| anyhow!("Failed to lock watcher mutex: {}", e))? = Some(watcher); - Ok(()) - } - - /// Process pending file system events from the watcher. - /// - /// This should be called periodically to sync external file changes into the VFS. - /// Returns the number of files that were updated. - pub fn process_file_events(&self) -> usize { - // Get events from the watcher - let events = { - let Ok(guard) = self.watcher.lock() else { - return 0; // Return 0 if mutex is poisoned - }; - if let Some(watcher) = guard.as_ref() { - watcher.try_recv_events() - } else { - return 0; - } - }; - - let mut updated_count = 0; - - for event in events { - match event { - WatchEvent::Modified(path) | WatchEvent::Created(path) => { - if let Err(e) = self.load_from_disk(&path) { - eprintln!("Failed to load file from disk: {path}: {e}"); - } else { - updated_count += 1; - } - } - WatchEvent::Deleted(path) => { - // For now, we don't remove deleted files from VFS - // This maintains stable `FileId`s for consumers - eprintln!("File deleted (keeping in VFS): {path}"); - } - WatchEvent::Renamed { from, to } => { - // Handle rename by updating the path mapping - if let Some(file_id) = self.by_path.remove(&from).map(|(_, id)| id) { - self.by_path.insert(to.clone(), file_id); - if let Err(e) = self.load_from_disk(&to) { - eprintln!("Failed to load renamed file: {to}: {e}"); - } else { - updated_count += 1; - } - } - } - } - } - updated_count - } - - /// Load a file's content from disk and update the VFS. - /// - /// This method reads the file from the filesystem and updates the VFS entry - /// if the content has changed. It's used by the file watcher to sync external changes. - fn load_from_disk(&self, path: &Utf8PathBuf) -> Result<()> { - // Check if we have this file tracked - if let Some(file_id) = self.by_path.get(path).map(|entry| *entry.value()) { - // Read content from disk - let content = fs::read_to_string(path.as_std_path()) - .map_err(|e| anyhow!("Failed to read file {}: {}", path, e))?; - - let new_text = TextSource::Disk(Arc::from(content.as_str())); - let new_hash = FileHash::from_text_source(&new_text); - - // Update the file if content changed - if let Some(mut record) = self.files.get_mut(&file_id) { - if new_hash.differs_from(record.hash) { - record.text = new_text; - record.hash = new_hash; - self.head.fetch_add(1, Ordering::SeqCst); - } - } - } - Ok(()) - } - - /// Check if file watching is currently enabled. - pub fn is_file_watching_enabled(&self) -> bool { - self.watcher.lock().map(|g| g.is_some()).unwrap_or(false) // Return false if mutex is poisoned - } -} - -impl Default for Vfs { - fn default() -> Self { - Self { - next_file_id: AtomicU32::new(0), - by_uri: DashMap::new(), - files: DashMap::new(), - head: AtomicU64::new(0), - watcher: std::sync::Mutex::new(None), - by_path: DashMap::new(), - } - } -} - -/// Immutable snapshot view of the VFS at a specific revision. -/// -/// [`VfsSnapshot`] provides a consistent view of all files for downstream consumers, -/// avoiding the need for locking during processing. Snapshots are created atomically -/// and can be safely shared across threads. -#[derive(Clone)] -pub(crate) struct VfsSnapshot { - /// The global revision at the time of snapshot - revision: Revision, - /// All files in the VFS at snapshot time - pub files: HashMap, -} - -impl VfsSnapshot { - /// Get the text content of a file in this snapshot. - /// - /// Returns `None` if the [`FileId`] is not present in the snapshot. - #[must_use] - pub fn get_text(&self, id: FileId) -> Option> { - self.files.get(&id).map(|r| match &r.text { - TextSource::Disk(s) | TextSource::Overlay(s) | TextSource::Generated(s) => s.clone(), - }) - } - - /// Get the metadata for a file in this snapshot. - /// - /// Returns `None` if the [`FileId`] is not present in the snapshot. - #[must_use] - pub fn meta(&self, id: FileId) -> Option<&FileMeta> { - self.files.get(&id).map(|r| &r.meta) - } -} diff --git a/crates/djls-workspace/src/vfs/watcher.rs b/crates/djls-workspace/src/vfs/watcher.rs deleted file mode 100644 index aee0467..0000000 --- a/crates/djls-workspace/src/vfs/watcher.rs +++ /dev/null @@ -1,325 +0,0 @@ -//! File system watching for VFS synchronization. -//! -//! This module provides file system watching capabilities to detect external changes -//! and synchronize them with the VFS. It uses cross-platform file watching with -//! debouncing to handle rapid changes efficiently. - -use std::collections::HashMap; -use std::sync::mpsc; -use std::thread; -use std::time::Duration; -use std::time::Instant; - -use anyhow::anyhow; -use anyhow::Result; -use camino::Utf8PathBuf; -use notify::Config; -use notify::Event; -use notify::EventKind; -use notify::RecommendedWatcher; -use notify::RecursiveMode; -use notify::Watcher; - -/// Event types that can occur in the file system. -/// -/// [`WatchEvent`] represents the different types of file system changes that -/// the watcher can detect and process. -#[derive(Clone, Debug, PartialEq)] -pub enum WatchEvent { - /// A file was modified (content changed) - Modified(Utf8PathBuf), - /// A new file was created - Created(Utf8PathBuf), - /// A file was deleted - Deleted(Utf8PathBuf), - /// A file was renamed from one path to another - Renamed { from: Utf8PathBuf, to: Utf8PathBuf }, -} - -/// Configuration for the file watcher. -/// -/// [`WatchConfig`] controls how the file watcher operates, including what -/// directories to watch and how to filter events. -#[derive(Clone, Debug)] -pub struct WatchConfig { - /// Whether file watching is enabled - pub enabled: bool, - /// Root directories to watch recursively - pub roots: Vec, - /// Debounce time in milliseconds (collect events for this duration before processing) - pub debounce_ms: u64, - /// File patterns to include (e.g., ["*.py", "*.html"]) - pub include_patterns: Vec, - /// File patterns to exclude (e.g., ["__pycache__", ".git", "*.pyc"]) - pub exclude_patterns: Vec, -} - -// TODO: Allow for user config instead of hardcoding defaults -impl Default for WatchConfig { - fn default() -> Self { - Self { - enabled: true, - roots: Vec::new(), - debounce_ms: 250, - include_patterns: vec!["*.py".to_string(), "*.html".to_string()], - exclude_patterns: vec![ - "__pycache__".to_string(), - ".git".to_string(), - ".pyc".to_string(), - "node_modules".to_string(), - ".venv".to_string(), - "venv".to_string(), - ], - } - } -} - -/// File system watcher for VFS synchronization. -/// -/// [`VfsWatcher`] monitors the file system for changes and provides a channel -/// for consuming batched events. It handles debouncing and filtering internally. -pub struct VfsWatcher { - /// The underlying file system watcher - _watcher: RecommendedWatcher, - /// Receiver for processed watch events - rx: mpsc::Receiver>, - /// Configuration for the watcher - config: WatchConfig, - /// Handle to the background processing thread - _handle: thread::JoinHandle<()>, -} - -impl VfsWatcher { - /// Create a new file watcher with the given configuration. - /// - /// This starts watching the specified root directories and begins processing - /// events in a background thread. - pub fn new(config: WatchConfig) -> Result { - if !config.enabled { - return Err(anyhow!("File watching is disabled")); - } - - let (event_tx, event_rx) = mpsc::channel(); - let (watch_tx, watch_rx) = mpsc::channel(); - - // Create the file system watcher - let mut watcher = RecommendedWatcher::new( - move |res: notify::Result| { - if let Ok(event) = res { - let _ = event_tx.send(event); - } - }, - Config::default(), - )?; - - // Watch all root directories - for root in &config.roots { - let std_path = root.as_std_path(); - if std_path.exists() { - watcher.watch(std_path, RecursiveMode::Recursive)?; - } - } - - // Spawn background thread for event processing - let config_clone = config.clone(); - let handle = thread::spawn(move || { - Self::process_events(&event_rx, &watch_tx, &config_clone); - }); - - Ok(Self { - _watcher: watcher, - rx: watch_rx, - config, - _handle: handle, - }) - } - - /// Get the next batch of processed watch events. - /// - /// This is a non-blocking operation that returns immediately. If no events - /// are available, it returns an empty vector. - #[must_use] - pub fn try_recv_events(&self) -> Vec { - self.rx.try_recv().unwrap_or_default() - } - - /// Background thread function for processing raw file system events. - /// - /// This function handles debouncing, filtering, and batching of events before - /// sending them to the main thread for VFS synchronization. - fn process_events( - event_rx: &mpsc::Receiver, - watch_tx: &mpsc::Sender>, - config: &WatchConfig, - ) { - let mut pending_events: HashMap = HashMap::new(); - let mut last_batch_time = Instant::now(); - let debounce_duration = Duration::from_millis(config.debounce_ms); - - loop { - // Try to receive events with a timeout for batching - match event_rx.recv_timeout(Duration::from_millis(50)) { - Ok(event) => { - // Process the raw notify event into our WatchEvent format - if let Some(watch_events) = Self::convert_notify_event(event, config) { - for watch_event in watch_events { - let path = Self::get_event_path(&watch_event); - // Only keep the latest event for each path - pending_events.insert(path.clone(), watch_event); - } - } - } - Err(mpsc::RecvTimeoutError::Timeout) => { - // Timeout - check if we should flush pending events - } - Err(mpsc::RecvTimeoutError::Disconnected) => { - // Channel disconnected, exit the thread - break; - } - } - - // Check if we should flush pending events - if !pending_events.is_empty() && last_batch_time.elapsed() >= debounce_duration { - let events: Vec = pending_events.values().cloned().collect(); - if watch_tx.send(events).is_err() { - // Main thread disconnected, exit - break; - } - pending_events.clear(); - last_batch_time = Instant::now(); - } - } - } - - /// Convert a [`notify::Event`] into our [`WatchEvent`] format. - fn convert_notify_event(event: Event, config: &WatchConfig) -> Option> { - let mut watch_events = Vec::new(); - - for path in event.paths { - if let Ok(utf8_path) = Utf8PathBuf::try_from(path) { - if Self::should_include_path_static(&utf8_path, config) { - match event.kind { - EventKind::Create(_) => watch_events.push(WatchEvent::Created(utf8_path)), - EventKind::Modify(_) => watch_events.push(WatchEvent::Modified(utf8_path)), - EventKind::Remove(_) => watch_events.push(WatchEvent::Deleted(utf8_path)), - _ => {} // Ignore other event types for now - } - } - } - } - - if watch_events.is_empty() { - None - } else { - Some(watch_events) - } - } - - /// Static version of should_include_path for use in convert_notify_event. - fn should_include_path_static(path: &Utf8PathBuf, config: &WatchConfig) -> bool { - let path_str = path.as_str(); - - // Check exclude patterns first - for pattern in &config.exclude_patterns { - if path_str.contains(pattern) { - return false; - } - } - - // If no include patterns, include everything (that's not excluded) - if config.include_patterns.is_empty() { - return true; - } - - // Check include patterns - for pattern in &config.include_patterns { - if let Some(extension) = pattern.strip_prefix("*.") { - if path_str.ends_with(extension) { - return true; - } - } else if path_str.contains(pattern) { - return true; - } - } - - false - } - - /// Extract the path from a [`WatchEvent`]. - fn get_event_path(event: &WatchEvent) -> &Utf8PathBuf { - match event { - WatchEvent::Modified(path) | WatchEvent::Created(path) | WatchEvent::Deleted(path) => { - path - } - WatchEvent::Renamed { to, .. } => to, - } - } -} - -impl Drop for VfsWatcher { - fn drop(&mut self) { - // The background thread will exit when the event channel is dropped - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_watch_config_default() { - let config = WatchConfig::default(); - assert!(config.enabled); - assert_eq!(config.debounce_ms, 250); - assert!(config.include_patterns.contains(&"*.py".to_string())); - assert!(config.exclude_patterns.contains(&".git".to_string())); - } - - #[test] - fn test_should_include_path() { - let config = WatchConfig::default(); - - // Should include Python files - assert!(VfsWatcher::should_include_path_static( - &Utf8PathBuf::from("test.py"), - &config - )); - - // Should include HTML files - assert!(VfsWatcher::should_include_path_static( - &Utf8PathBuf::from("template.html"), - &config - )); - - // Should exclude .git files - assert!(!VfsWatcher::should_include_path_static( - &Utf8PathBuf::from(".git/config"), - &config - )); - - // Should exclude __pycache__ files - assert!(!VfsWatcher::should_include_path_static( - &Utf8PathBuf::from("__pycache__/test.pyc"), - &config - )); - } - - #[test] - fn test_watch_event_types() { - let path1 = Utf8PathBuf::from("test.py"); - let path2 = Utf8PathBuf::from("new.py"); - - let modified = WatchEvent::Modified(path1.clone()); - let created = WatchEvent::Created(path1.clone()); - let deleted = WatchEvent::Deleted(path1.clone()); - let renamed = WatchEvent::Renamed { - from: path1, - to: path2, - }; - - // Test that events can be created and compared - assert_ne!(modified, created); - assert_ne!(created, deleted); - assert_ne!(deleted, renamed); - } -} From 96e0b814179cfa6793d42cd38043e31bc6986b9e Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 28 Aug 2025 22:54:41 -0500 Subject: [PATCH 13/30] wowza --- ARCHITECTURE_INSIGHTS.md | 96 +++ REVISION_TRACKING_ARCHITECTURE.md | 341 ++++++++++ RUFF_ARCHITECTURE_INSIGHTS.md | 77 +++ check_ruff_pattern.md | 94 +++ crates/djls-server/src/server.rs | 100 ++- crates/djls-server/src/session.rs | 637 ++++++++++++++++-- crates/djls-workspace/src/bridge.rs | 126 ---- crates/djls-workspace/src/db.rs | 415 +++++++----- crates/djls-workspace/src/document.rs | 216 ++++++ .../djls-workspace/src/document/line_index.rs | 90 --- crates/djls-workspace/src/document/mod.rs | 130 ---- crates/djls-workspace/src/fs.rs | 269 ++++++++ .../src/{document => }/language.rs | 6 +- crates/djls-workspace/src/lib.rs | 49 +- crates/djls-workspace/src/lsp_system.rs | 154 ----- crates/djls-workspace/src/system.rs | 118 ---- .../src/{document => }/template.rs | 0 crates/djls-workspace/src/test_db.rs | 25 - task_order.md | 61 ++ 19 files changed, 2091 insertions(+), 913 deletions(-) create mode 100644 ARCHITECTURE_INSIGHTS.md create mode 100644 REVISION_TRACKING_ARCHITECTURE.md create mode 100644 RUFF_ARCHITECTURE_INSIGHTS.md create mode 100644 check_ruff_pattern.md delete mode 100644 crates/djls-workspace/src/bridge.rs create mode 100644 crates/djls-workspace/src/document.rs delete mode 100644 crates/djls-workspace/src/document/line_index.rs delete mode 100644 crates/djls-workspace/src/document/mod.rs create mode 100644 crates/djls-workspace/src/fs.rs rename crates/djls-workspace/src/{document => }/language.rs (79%) delete mode 100644 crates/djls-workspace/src/lsp_system.rs delete mode 100644 crates/djls-workspace/src/system.rs rename crates/djls-workspace/src/{document => }/template.rs (100%) delete mode 100644 crates/djls-workspace/src/test_db.rs create mode 100644 task_order.md diff --git a/ARCHITECTURE_INSIGHTS.md b/ARCHITECTURE_INSIGHTS.md new file mode 100644 index 0000000..b3dcdf0 --- /dev/null +++ b/ARCHITECTURE_INSIGHTS.md @@ -0,0 +1,96 @@ +# Architecture Insights from Ruff Investigation + +## Key Discovery: Two-Layer Architecture + +### The Problem +- LSP documents change frequently (every keystroke) +- Salsa invalidation is expensive +- Tower-lsp requires Send+Sync, but Salsa Database contains RefCell/UnsafeCell + +### The Solution (Ruff Pattern) + +#### Layer 1: LSP Document Management (Outside Salsa) +- Store overlays in `Session` using `Arc>` +- TextDocument contains actual content, version, language_id +- Changes are immediate, no Salsa invalidation + +#### Layer 2: Salsa Incremental Computation +- Database is pure Salsa, no file storage +- Queries read through FileSystem trait +- LspFileSystem intercepts reads, returns overlay or disk content + +### Critical Insights + +1. **Overlays NEVER become Salsa inputs directly** + - They're intercepted at FileSystem::read_to_string() time + - Salsa only knows "something changed", reads content lazily + +2. **StorageHandle Pattern (for tower-lsp)** + - Session stores `StorageHandle` not Database directly + - StorageHandle IS Send+Sync even though Database isn't + - Create Database instances on-demand: `session.db()` + +3. **File Management Location** + - WRONG: Store files in Database (what we initially did) + - RIGHT: Store overlays in Session, Database is pure Salsa + +4. **The Bridge** + - LspFileSystem has Arc to same overlays as Session + - When Salsa queries need content, they call FileSystem + - FileSystem checks overlays first, falls back to disk + +### Implementation Flow + +1. **did_open/did_change/did_close** → Update overlays in Session +2. **notify_file_changed()** → Tell Salsa something changed +3. **Salsa query executes** → Calls FileSystem::read_to_string() +4. **LspFileSystem intercepts** → Returns overlay if exists, else disk +5. **Query gets content** → Without knowing about LSP/overlays + +### Why This Works + +- Fast: Overlay updates don't trigger Salsa invalidation cascade +- Thread-safe: DashMap for overlays, StorageHandle for Database +- Clean separation: LSP concerns vs computation concerns +- Efficient: Salsa caching still works, just reads through FileSystem + +### Tower-lsp vs lsp-server + +- **Ruff uses lsp-server**: No Send+Sync requirement, can store Database directly +- **We use tower-lsp**: Requires Send+Sync, must use StorageHandle pattern +- Both achieve same result, different mechanisms + +## Critical Implementation Details (From Ruff Expert) + +### The Revision Dependency Trick + +**THE MOST CRITICAL INSIGHT**: In the `source_text` tracked function, calling `file.revision(db)` is what creates the Salsa dependency chain: + +```rust +#[salsa::tracked] +pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { + // THIS LINE IS CRITICAL - Creates Salsa dependency on revision! + let _ = file.revision(db); + + // Now read from FileSystem (checks overlays first) + db.read_file_content(file.path(db)) +} +``` + +Without that `file.revision(db)` call, revision changes won't trigger invalidation! + +### Key Implementation Points + +1. **Files have no text**: SourceFile inputs only have `path` and `revision`, never `text` +2. **Revision bumping triggers invalidation**: Change revision → source_text invalidated → dependent queries invalidated +3. **Files created lazily**: Don't pre-create, let them be created on first access +4. **Simple counters work**: Revision can be a simple u64 counter, doesn't need timestamps +5. **StorageHandle update required**: After DB modifications in LSP handlers, must update the handle + +### Common Pitfalls + +- **Forgetting the revision dependency** - Without `file.revision(db)`, nothing invalidates +- **Storing text in Salsa inputs** - Breaks the entire pattern +- **Not bumping revision on overlay changes** - Queries won't see new content +- **Creating files eagerly** - Unnecessary and inefficient + diff --git a/REVISION_TRACKING_ARCHITECTURE.md b/REVISION_TRACKING_ARCHITECTURE.md new file mode 100644 index 0000000..202c118 --- /dev/null +++ b/REVISION_TRACKING_ARCHITECTURE.md @@ -0,0 +1,341 @@ +# Revision Tracking Architecture for Django Language Server + +## Overview + +This document captures the complete understanding of how to implement revision tracking for task-112, based on extensive discussions with a Ruff architecture expert. The goal is to connect the Session's overlay system with Salsa's query invalidation mechanism through per-file revision tracking. + +## The Critical Breakthrough: Dual-Layer Architecture + +### The Confusion We Had + +We conflated two different concepts: +1. **Database struct** - The Rust struct that implements the Salsa database trait +2. **Salsa database** - The actual Salsa storage system with inputs/queries + +### The Key Insight + +**Database struct ≠ Salsa database** + +The Database struct can contain: +- Salsa storage (the actual Salsa database) +- Additional non-Salsa data structures (like file tracking) + +## The Architecture Pattern (From Ruff) + +### Ruff's Implementation + +```rust +// Ruff's Database contains BOTH Salsa and non-Salsa data +pub struct ProjectDatabase { + storage: salsa::Storage, // Salsa's data + files: Files, // NOT Salsa data, but in Database struct! +} + +// Files is Arc-wrapped for cheap cloning +#[derive(Clone)] +pub struct Files { + inner: Arc, // Shared across clones +} + +struct FilesInner { + system_by_path: FxDashMap, // Thread-safe +} +``` + +### Our Implementation + +```rust +// Django LS Database structure +#[derive(Clone)] +pub struct Database { + storage: salsa::Storage, + files: Arc>, // Arc makes cloning cheap! +} + +// Session still uses StorageHandle for tower-lsp +pub struct Session { + db_handle: StorageHandle, // Still needed! + overlays: Arc>, // LSP document state +} +``` + +## Why This Works with Send+Sync Requirements + +1. **Arc is Send+Sync** - Thread-safe by design +2. **Cloning is cheap** - Only clones the Arc pointer (8 bytes) +3. **Persistence across clones** - All clones share the same DashMap +4. **StorageHandle compatible** - Database remains clonable and Send+Sync + +## Implementation Details + +### 1. Database Implementation + +```rust +impl Database { + pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { + self.files + .entry(path.clone()) + .or_insert_with(|| { + // Create Salsa input with initial revision 0 + SourceFile::new(self, path, 0) + }) + .clone() + } +} + +impl Clone for Database { + fn clone(&self) -> self { + Self { + storage: self.storage.clone(), // Salsa handles this + files: self.files.clone(), // Just clones Arc! + } + } +} +``` + +### 2. The Critical Pattern for Every Overlay Change + +```rust +pub fn handle_overlay_change(session: &mut Session, url: Url, content: String) { + // 1. Extract database from StorageHandle + let mut db = session.db_handle.get(); + + // 2. Update overlay in Session + session.overlays.insert(url.clone(), TextDocument::new(content)); + + // 3. Get or create file in Database + let path = path_from_url(&url); + let file = db.get_or_create_file(path); + + // 4. Bump revision (simple incrementing counter) + let current_rev = file.revision(&db); + file.set_revision(&mut db).to(current_rev + 1); + + // 5. Update StorageHandle with modified database + session.db_handle.update(db); // CRITICAL! +} +``` + +### 3. LSP Handler Updates + +#### did_open + +```rust +pub fn did_open(&mut self, params: DidOpenTextDocumentParams) { + let mut db = self.session.db_handle.get(); + + // Set overlay + self.session.overlays.insert( + params.text_document.uri.clone(), + TextDocument::new(params.text_document.text) + ); + + // Create file with initial revision 0 + let path = path_from_url(¶ms.text_document.uri); + db.get_or_create_file(path); // Creates with revision 0 + + self.session.db_handle.update(db); +} +``` + +#### did_change + +```rust +pub fn did_change(&mut self, params: DidChangeTextDocumentParams) { + let mut db = self.session.db_handle.get(); + + // Update overlay + let new_content = params.content_changes[0].text.clone(); + self.session.overlays.insert( + params.text_document.uri.clone(), + TextDocument::new(new_content) + ); + + // Bump revision + let path = path_from_url(¶ms.text_document.uri); + let file = db.get_or_create_file(path); + let new_rev = file.revision(&db) + 1; + file.set_revision(&mut db).to(new_rev); + + self.session.db_handle.update(db); +} +``` + +#### did_close + +```rust +pub fn did_close(&mut self, params: DidCloseTextDocumentParams) { + let mut db = self.session.db_handle.get(); + + // Remove overlay + self.session.overlays.remove(¶ms.text_document.uri); + + // Bump revision to trigger re-read from disk + let path = path_from_url(¶ms.text_document.uri); + if let Some(file) = db.files.get(&path) { + let new_rev = file.revision(&db) + 1; + file.set_revision(&mut db).to(new_rev); + } + + self.session.db_handle.update(db); +} +``` + +## Key Implementation Guidelines from Ruff Expert + +### 1. File Tracking Location + +- Store in Database struct (not Session) +- Use Arc for thread-safety and cheap cloning +- This keeps file tracking close to where it's used + +### 2. Revision Management + +- Use simple incrementing counter per file (not timestamps) +- Each file has independent revision tracking +- Revision just needs to change, doesn't need to be monotonic +- Example: `file.set_revision(&mut db).to(current + 1)` + +### 3. Lazy File Creation + +Files should be created: +- On did_open (via get_or_create_file) +- On first query access if needed +- NOT eagerly for all possible files + +### 4. File Lifecycle + +- **On open**: Create file with revision 0 +- **On change**: Bump revision to trigger invalidation +- **On close**: Keep file alive, bump revision for re-read from disk +- **Never remove**: Files stay in tracking even after close + +### 5. Batch Changes for Performance + +When possible, batch multiple changes: + +```rust +pub fn apply_batch_changes(&mut self, changes: Vec) { + let mut db = self.session.db_handle.get(); + + for change in changes { + // Process each change + let file = db.get_or_create_file(change.path); + file.set_revision(&mut db).to(file.revision(&db) + 1); + } + + // Single StorageHandle update at the end + self.session.db_handle.update(db); +} +``` + +### 6. Thread Safety with DashMap + +Use DashMap's atomic entry API: + +```rust +self.files.entry(path.clone()) + .and_modify(|file| { + // Modify existing + file.set_revision(db).to(new_rev); + }) + .or_insert_with(|| { + // Create new + SourceFile::builder(path) + .revision(0) + .new(db) + }); +``` + +## Critical Pitfalls to Avoid + +1. **NOT BUMPING REVISION** - Every overlay change MUST bump revision or Salsa won't invalidate +2. **FORGETTING STORAGEHANDLE UPDATE** - Must call `session.db_handle.update(db)` after changes +3. **CREATING FILES EAGERLY** - Let files be created lazily on first access +4. **USING TIMESTAMPS** - Simple incrementing counter is sufficient +5. **REMOVING FILES** - Keep files alive even after close, just bump revision + +## The Two-Layer Model + +### Layer 1: Non-Salsa (but in Database struct) +- `Arc>` - File tracking +- Thread-safe via Arc+DashMap +- Cheap to clone via Arc +- Acts as a lookup table + +### Layer 2: Salsa Inputs +- `SourceFile` entities created via `SourceFile::new(db)` +- Have revision fields for invalidation +- Tracked by Salsa's dependency system +- Invalidation cascades through dependent queries + +## Complete Architecture Summary + +| Component | Contains | Purpose | +|-----------|----------|---------| +| **Database** | `storage` + `Arc>` | Salsa queries + file tracking | +| **Session** | `StorageHandle` + `Arc>` | LSP state + overlays | +| **StorageHandle** | `Arc>>` | Bridge for tower-lsp lifetime requirements | +| **SourceFile** | Salsa input with path + revision | Triggers query invalidation | + +## The Flow + +1. **LSP request arrives** → tower-lsp handler +2. **Extract database** → `db = session.db_handle.get()` +3. **Update overlay** → `session.overlays.insert(url, content)` +4. **Get/create file** → `db.get_or_create_file(path)` +5. **Bump revision** → `file.set_revision(&mut db).to(current + 1)` +6. **Update handle** → `session.db_handle.update(db)` +7. **Salsa invalidates** → `source_text` query re-executes +8. **Queries see new content** → Through overlay-aware FileSystem + +## Why StorageHandle is Still Essential + +1. **tower-lsp requirement**: Needs 'static lifetime for async handlers +2. **Snapshot management**: Safe extraction and update of database +3. **Thread safety**: Bridges async boundaries safely +4. **Atomic updates**: Ensures consistent state transitions + +## Testing Strategy + +1. **Revision bumping**: Verify each overlay operation bumps revision +2. **Invalidation cascade**: Ensure source_text re-executes after revision bump +3. **Thread safety**: Concurrent overlay updates work correctly +4. **Clone behavior**: Database clones share the same file tracking +5. **Lazy creation**: Files only created when accessed + +## Implementation Checklist + +- [ ] Add `Arc>` to Database struct +- [ ] Implement Clone for Database (clone both storage and Arc) +- [ ] Create `get_or_create_file` method using atomic entry API +- [ ] Update did_open to create files with revision 0 +- [ ] Update did_change to bump revision after overlay update +- [ ] Update did_close to bump revision (keep file alive) +- [ ] Ensure StorageHandle updates after all database modifications +- [ ] Add tests for revision tracking and invalidation + +## Questions That Were Answered + +1. **Q: Files in Database or Session?** + A: In Database, but Arc-wrapped for cheap cloning + +2. **Q: How does this work with Send+Sync?** + A: Arc is Send+Sync, making Database clonable and thread-safe + +3. **Q: Do we still need StorageHandle?** + A: YES! It bridges tower-lsp's lifetime requirements + +4. **Q: Timestamp or counter for revisions?** + A: Simple incrementing counter per file + +5. **Q: Remove files on close?** + A: No, keep them alive and bump revision for re-read + +## The Key Insight + +Database struct is a container that holds BOTH: +- Salsa storage (for queries and inputs) +- Non-Salsa data (file tracking via Arc) + +Arc makes the non-Salsa data cheap to clone while maintaining Send+Sync compatibility. This is the pattern Ruff uses and what we should implement. \ No newline at end of file diff --git a/RUFF_ARCHITECTURE_INSIGHTS.md b/RUFF_ARCHITECTURE_INSIGHTS.md new file mode 100644 index 0000000..f4a0bf1 --- /dev/null +++ b/RUFF_ARCHITECTURE_INSIGHTS.md @@ -0,0 +1,77 @@ +# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution + +## This document is preserved for historical context but is OUTDATED +## We found the StorageHandle solution that solves the Send+Sync issue + +# Critical Discovery: The Tower-LSP vs lsp-server Architectural Mismatch + +## The Real Problem + +Your Ruff expert friend is correct. The fundamental issue is: + +### What We Found: + +1. **Salsa commit a3ffa22 uses `RefCell` and `UnsafeCell`** - These are inherently not `Sync` +2. **Tower-LSP requires `Sync`** - Because handlers take `&self` in async contexts +3. **Ruff uses `lsp-server`** - Which doesn't require `Sync` on the server struct + +### The Mystery: + +Your expert suggests Ruff's database IS `Send + Sync`, but our testing shows that with the same Salsa commit, the database contains: +- `RefCell` (not Sync) +- `UnsafeCell>` (not Sync) + +## Possible Explanations: + +### Theory 1: Ruff Has Custom Patches +Ruff might have additional patches or workarounds not visible in the commit hash. + +### Theory 2: Different Usage Pattern +Ruff might structure their database differently to avoid the Sync requirement entirely. + +### Theory 3: lsp-server Architecture +Since Ruff uses `lsp-server` (not `tower-lsp`), they might never need the database to be Sync: +- They clone the database for background work (requires Send only) +- The main thread owns the database, background threads get clones +- No shared references across threads + +## Verification Needed: + +1. **Check if Ruff's database is actually Sync**: + - Look for unsafe impl Sync in their codebase + - Check if they wrap the database differently + +2. **Understand lsp-server's threading model**: + - How does it handle async without requiring Sync? + - What's the message passing pattern? + +## Solution Decision Matrix (Updated): + +| Solution | Effort | Performance | Risk | Compatibility | +|----------|---------|------------|------|---------------| +| **Switch to lsp-server** | High | High | Medium | Perfect Ruff parity | +| **Actor Pattern** | Medium | Medium | Low | Works with tower-lsp | +| **Arc** | Low | Poor | Low | Works but slow | +| **Unsafe Sync wrapper** | Low | High | Very High | Dangerous | +| **Database per request** | Medium | Poor | Low | Loses memoization | + +## Recommended Action Plan: + +### Immediate (Today): +1. Verify that Salsa a3ffa22 truly has RefCell/UnsafeCell +2. Check if there are any Ruff-specific patches to Salsa +3. Test the actor pattern as a better alternative to Arc + +### Short-term (This Week): +1. Implement actor pattern if Salsa can't be made Sync +2. OR investigate unsafe Sync wrapper with careful single-threaded access guarantees + +### Long-term (This Month): +1. Consider migrating to lsp-server for full Ruff compatibility +2. OR contribute Sync support to Salsa upstream + +## The Key Insight: + +**Tower-LSP's architecture is fundamentally incompatible with Salsa's current design.** + +Ruff avoided this by using `lsp-server`, which has a different threading model that doesn't require Sync on the database. diff --git a/check_ruff_pattern.md b/check_ruff_pattern.md new file mode 100644 index 0000000..5253b69 --- /dev/null +++ b/check_ruff_pattern.md @@ -0,0 +1,94 @@ +# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution + +## This document is preserved for historical context but is OUTDATED +## We found the StorageHandle solution that solves the Send+Sync issue + +# Key Findings from Ruff's Architecture + +Based on the exploration, here's what we discovered: + +## Current Django LS Architecture + +### What We Have: +1. `Database` struct with `#[derive(Clone)]` and Salsa storage +2. `WorkspaceDatabase` that wraps `Database` and uses `DashMap` for thread-safe file storage +3. `Session` that owns `WorkspaceDatabase` directly (not wrapped in Arc) +4. Tower-LSP server that requires `Send + Sync` for async handlers + +### The Problem: +- `Database` is not `Sync` due to `RefCell` and `UnsafeCell` in Salsa's `ZalsaLocal` +- This prevents `Session` from being `Sync`, which breaks tower-lsp async handlers + +## Ruff's Solution (From Analysis) + +### They Don't Make Database Sync! +The key insight is that Ruff **doesn't actually make the database Send + Sync**. Instead: + +1. **Clone for Background Work**: They clone the database for each background task +2. **Move Not Share**: The cloned database is *moved* into the task (requires Send, not Sync) +3. **Message Passing**: Results are sent back via channels + +### Critical Difference: +- Ruff uses a custom LSP implementation that doesn't require `Sync` on the session +- Tower-LSP *does* require `Sync` because handlers take `&self` + +## The Real Problem + +Tower-LSP's `LanguageServer` trait requires: +```rust +async fn initialize(&self, ...) -> ... +// ^^^^^ This requires self to be Sync! +``` + +But with Salsa's current implementation, the Database can never be Sync. + +## Solution Options + +### Option 1: Wrap Database in Arc (Current Workaround) +```rust +pub struct Session { + database: Arc>, + // ... +} +``` +Downsides: Lock contention, defeats purpose of Salsa's internal optimization + +### Option 2: Move Database Out of Session +```rust +pub struct Session { + // Don't store database here + file_index: Arc>, + settings: Settings, +} + +// Create database on demand for each request +impl LanguageServer for Server { + async fn some_handler(&self) { + let db = create_database_from_index(&self.session.file_index); + // Use db for this request + } +} +``` + +### Option 3: Use Actor Pattern +```rust +pub struct DatabaseActor { + database: WorkspaceDatabase, + rx: mpsc::Receiver, +} + +pub struct Session { + db_tx: mpsc::Sender, +} +``` + +### Option 4: Custom unsafe Send/Sync implementation +This is risky but possible if we ensure single-threaded access patterns. + +## The Salsa Version Mystery + +We're using the exact same Salsa commit as Ruff, with the same features. The issue is NOT the Salsa version, but how tower-lsp forces us to use it. + +Ruff likely either: +1. Doesn't use tower-lsp (has custom implementation) +2. Or structures their server differently to avoid needing Sync on the database diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index e002658..daa64ac 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -6,6 +6,7 @@ use tower_lsp_server::jsonrpc::Result as LspResult; use tower_lsp_server::lsp_types; use tower_lsp_server::LanguageServer; use tracing_appender::non_blocking::WorkerGuard; +use url::Url; use crate::queue::Queue; use crate::session::Session; @@ -202,13 +203,19 @@ impl LanguageServer for DjangoLanguageServer { async fn did_open(&self, params: lsp_types::DidOpenTextDocumentParams) { tracing::info!("Opened document: {:?}", params.text_document.uri); - self.with_session_mut(|_session| { - // TODO: Handle document open after refactoring - let _uri = params.text_document.uri.clone(); - let _version = params.text_document.version; - let _language_id = + self.with_session_mut(|session| { + // Convert LSP types to our types + let url = + Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let language_id = djls_workspace::LanguageId::from(params.text_document.language_id.as_str()); - let _text = params.text_document.text.clone(); + let document = djls_workspace::TextDocument::new( + params.text_document.text, + params.text_document.version, + language_id, + ); + + session.open_document(url, document); }) .await; } @@ -216,11 +223,29 @@ impl LanguageServer for DjangoLanguageServer { async fn did_change(&self, params: lsp_types::DidChangeTextDocumentParams) { tracing::info!("Changed document: {:?}", params.text_document.uri); - self.with_session_mut(|_session| { - // TODO: Handle document change after refactoring - let _uri = ¶ms.text_document.uri; - let _version = params.text_document.version; - let _changes = params.content_changes.clone(); + self.with_session_mut(|session| { + let url = + Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + let new_version = params.text_document.version; + let changes = params.content_changes; + + if let Some(mut document) = session.get_overlay(&url) { + document.update(changes, new_version); + session.update_document(url, document); + } else { + // No existing overlay - shouldn't normally happen + tracing::warn!("Received change for document without overlay: {}", url); + + // Handle full content changes only for recovery + if let Some(change) = changes.into_iter().next() { + let document = djls_workspace::TextDocument::new( + change.text, + new_version, + djls_workspace::LanguageId::Other, + ); + session.update_document(url, document); + } + } }) .await; } @@ -228,19 +253,60 @@ impl LanguageServer for DjangoLanguageServer { async fn did_close(&self, params: lsp_types::DidCloseTextDocumentParams) { tracing::info!("Closed document: {:?}", params.text_document.uri); - self.with_session_mut(|_session| { - // TODO: Handle document close after refactoring - let _uri = ¶ms.text_document.uri; + self.with_session_mut(|session| { + let url = + Url::parse(¶ms.text_document.uri.to_string()).expect("Valid URI from LSP"); + + if session.close_document(&url).is_none() { + tracing::warn!("Attempted to close document without overlay: {}", url); + } }) .await; } async fn completion( &self, - _params: lsp_types::CompletionParams, + params: lsp_types::CompletionParams, ) -> LspResult> { - // TODO: Handle completion after refactoring - Ok(None) + let response = self + .with_session(|session| { + let lsp_uri = ¶ms.text_document_position.text_document.uri; + let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); + let position = params.text_document_position.position; + + tracing::debug!("Completion requested for {} at {:?}", url, position); + + // Check if we have an overlay for this document + if let Some(document) = session.get_overlay(&url) { + tracing::debug!("Using overlay content for completion in {}", url); + + // Use the overlay content for completion + // For now, we'll return None, but this is where completion logic would go + // The key point is that we're using overlay content, not disk content + let _content = document.content(); + let _version = document.version(); + + // TODO: Implement actual completion logic using overlay content + // This would involve: + // 1. Getting context around the cursor position + // 2. Analyzing the Django template or Python content + // 3. Returning appropriate completions + + None + } else { + tracing::debug!("No overlay found for {}, using disk content", url); + + // No overlay - would use disk content via the file system + // The LspFileSystem will automatically fall back to disk + // when no overlay is available + + // TODO: Implement completion using file system content + None + } + }) + .await; + + Ok(response) } async fn did_change_configuration(&self, _params: lsp_types::DidChangeConfigurationParams) { diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index beae4ec..6b4adbb 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,15 +1,75 @@ -use std::collections::HashMap; -use std::path::PathBuf; +//! # Salsa StorageHandle Pattern for LSP +//! +//! This module implements a thread-safe Salsa database wrapper for use with +//! tower-lsp's async runtime. The key challenge is that tower-lsp requires +//! `Send + Sync + 'static` bounds, but Salsa's `Storage` contains thread-local +//! state and is not `Send`. +//! +//! ## The Solution: StorageHandle +//! +//! Salsa provides `StorageHandle` which IS `Send + Sync` because it contains +//! no thread-local state. We store the handle and create `Storage`/`Database` +//! instances on-demand. +//! +//! ## The Mutation Challenge +//! +//! When mutating Salsa inputs (e.g., updating file revisions), Salsa must +//! ensure exclusive access to prevent race conditions. It does this via +//! `cancel_others()` which: +//! +//! 1. Sets a cancellation flag (causes other threads to panic with `Cancelled`) +//! 2. Waits for all `StorageHandle` clones to drop +//! 3. Proceeds with the mutation +//! +//! If we accidentally clone the handle instead of taking ownership, step 2 +//! never completes → deadlock! +//! +//! ## The Pattern +//! +//! - **Reads**: Clone the handle freely (`with_db`) +//! - **Mutations**: Take exclusive ownership (`with_db_mut` via `take_db_handle_for_mutation`) +//! +//! The explicit method names make the intent clear and prevent accidental misuse. + +use std::path::{Path, PathBuf}; use std::sync::Arc; +use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::{FileSystem, StdFileSystem, db::Database}; +use djls_workspace::{ + db::{Database, SourceFile}, + FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, +}; use percent_encoding::percent_decode_str; -use salsa::StorageHandle; +use salsa::{Setter, StorageHandle}; use tower_lsp_server::lsp_types; use url::Url; +/// LSP Session with thread-safe Salsa database access. +/// +/// Uses Salsa's `StorageHandle` pattern to maintain `Send + Sync + 'static` +/// compatibility required by tower-lsp. The handle can be safely shared +/// across threads and async boundaries. +/// +/// See [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) +/// for more information about `StorageHandle`. +/// +/// ## Architecture +/// +/// Two-layer system inspired by Ruff/Ty: +/// - **Layer 1**: In-memory overlays (LSP document edits) +/// - **Layer 2**: Salsa database (incremental computation cache) +/// +/// ## Salsa Mutation Protocol +/// +/// When mutating Salsa inputs (like changing file revisions), we must ensure +/// exclusive access to prevent race conditions. Salsa enforces this through +/// its `cancel_others()` mechanism, which waits for all `StorageHandle` clones +/// to drop before allowing mutations. +/// +/// We use explicit methods (`take_db_handle_for_mutation`/`restore_db_handle`) +/// to make this ownership transfer clear and prevent accidental deadlocks. pub struct Session { /// The Django project configuration project: Option, @@ -17,48 +77,82 @@ pub struct Session { /// LSP server settings settings: Settings, - /// A thread-safe Salsa database handle that can be shared between threads. + /// Layer 1: Thread-safe overlay storage (Arc>) /// - /// This implements the insight from [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) - /// where we're using the `StorageHandle` to create a thread-safe handle that can be - /// shared between threads. When we need to use it, we clone the handle to get a new reference. + /// This implements Ruff's two-layer architecture where Layer 1 contains + /// LSP overlays that take precedence over disk files. The overlays map + /// document URLs to TextDocuments containing current in-memory content. /// - /// This handle allows us to create database instances as needed. - /// Even though we're using a single-threaded runtime, we still need - /// this to be thread-safe because of LSP trait requirements. - /// - /// Usage: - /// ```rust,ignore - /// // Clone the StorageHandle for use in an async context - /// let db_handle = session.db_handle.clone(); - /// - /// // Use it in an async context - /// async_fn(move || { - /// // Get a database from the handle - /// let storage = db_handle.into_storage(); - /// let db = Database::from_storage(storage); - /// - /// // Use the database - /// db.some_query(args) - /// }); - /// ``` - db_handle: StorageHandle, + /// Key properties: + /// - Thread-safe via Arc for Send+Sync requirements + /// - Contains full TextDocument with content, version, and metadata + /// - Never becomes Salsa inputs - only intercepted at read time + overlays: Arc>, - /// File system abstraction for reading files + /// File system abstraction with overlay interception + /// + /// This LspFileSystem bridges Layer 1 (overlays) and Layer 2 (Salsa). + /// It intercepts FileSystem::read_to_string() calls to return overlay + /// content when available, falling back to disk otherwise. file_system: Arc, - /// Index of open documents with overlays (in-memory changes) - /// Maps document URL to its current content - overlays: HashMap, - - /// Tracks the session revision for change detection - revision: u64, + /// Shared file tracking across all Database instances + /// + /// This is the canonical Salsa pattern from the lazy-input example. + /// The DashMap provides O(1) lookups and is shared via Arc across + /// all Database instances created from StorageHandle. + files: Arc>, #[allow(dead_code)] client_capabilities: lsp_types::ClientCapabilities, + + /// Layer 2: Thread-safe Salsa database handle for pure computation + /// + /// where we're using the `StorageHandle` to create a thread-safe handle that can be + /// shared between threads. + /// + /// The database receives file content via the FileSystem trait, which + /// is intercepted by our LspFileSystem to provide overlay content. + /// This maintains proper separation between Layer 1 and Layer 2. + db_handle: StorageHandle, } impl Session { + pub fn new(params: &lsp_types::InitializeParams) -> Self { + let project_path = Self::get_project_path(params); + + let (project, settings) = if let Some(path) = &project_path { + let settings = + djls_conf::Settings::new(path).unwrap_or_else(|_| djls_conf::Settings::default()); + + let project = Some(djls_project::DjangoProject::new(path.clone())); + + (project, settings) + } else { + (None, Settings::default()) + }; + + let overlays = Arc::new(DashMap::new()); + let files = Arc::new(DashMap::new()); + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(OsFileSystem), + )); + let db_handle = Database::new(file_system.clone(), files.clone()) + .storage() + .clone() + .into_zalsa_handle(); + + Self { + project, + settings, + overlays, + file_system, + files, + client_capabilities: params.capabilities.clone(), + db_handle, + } + } /// Determines the project root path from initialization parameters. /// /// Tries the current directory first, then falls back to the first workspace folder. @@ -97,31 +191,6 @@ impl Session { Some(PathBuf::from(path_str)) } - pub fn new(params: &lsp_types::InitializeParams) -> Self { - let project_path = Self::get_project_path(params); - - let (project, settings) = if let Some(path) = &project_path { - let settings = - djls_conf::Settings::new(path).unwrap_or_else(|_| djls_conf::Settings::default()); - - let project = Some(djls_project::DjangoProject::new(path.clone())); - - (project, settings) - } else { - (None, Settings::default()) - }; - - Self { - client_capabilities: params.capabilities.clone(), - project, - settings, - db_handle: StorageHandle::new(None), - file_system: Arc::new(StdFileSystem), - overlays: HashMap::new(), - revision: 0, - } - } - pub fn project(&self) -> Option<&DjangoProject> { self.project.as_ref() } @@ -130,8 +199,6 @@ impl Session { &mut self.project } - - pub fn settings(&self) -> &Settings { &self.settings } @@ -144,23 +211,457 @@ impl Session { /// /// This creates a usable database from the handle, which can be used /// to query and update data. The database itself is not Send/Sync, - /// but the StorageHandle is, allowing us to work with tower-lsp. + /// but the `StorageHandle` is, allowing us to work with tower-lsp-server. + /// + /// The database will read files through the LspFileSystem, which + /// automatically returns overlay content when available. + /// + /// CRITICAL: We pass the shared files Arc to preserve file tracking + /// across Database reconstructions from StorageHandle. + #[allow(dead_code)] pub fn db(&self) -> Database { let storage = self.db_handle.clone().into_storage(); - Database::from_storage(storage) + Database::from_storage(storage, self.file_system.clone(), self.files.clone()) + } + + /// Get access to the file system (for Salsa integration) + #[allow(dead_code)] + pub fn file_system(&self) -> Arc { + self.file_system.clone() + } + + /// Set or update an overlay for the given document URL + /// + /// This implements Layer 1 of Ruff's architecture - storing in-memory + /// document changes that take precedence over disk content. + #[allow(dead_code)] // Used in tests + pub fn set_overlay(&self, url: Url, document: TextDocument) { + self.overlays.insert(url, document); + } + + /// Remove an overlay for the given document URL + /// + /// After removal, file reads will fall back to disk content. + #[allow(dead_code)] // Used in tests + pub fn remove_overlay(&self, url: &Url) -> Option { + self.overlays.remove(url).map(|(_, doc)| doc) + } + + /// Check if an overlay exists for the given URL + #[allow(dead_code)] + pub fn has_overlay(&self, url: &Url) -> bool { + self.overlays.contains_key(url) + } + + /// Get a copy of an overlay document + pub fn get_overlay(&self, url: &Url) -> Option { + self.overlays.get(url).map(|doc| doc.clone()) + } + + /// Takes exclusive ownership of the database handle for mutation operations. + /// + /// This method extracts the `StorageHandle` from the session, replacing it + /// with a temporary placeholder. This ensures there's exactly one handle + /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. + /// + /// # Why Not Clone? + /// + /// Cloning would create multiple handles. When Salsa needs to mutate inputs, + /// it calls `cancel_others()` which waits for all handles to drop. With + /// multiple handles, this wait would never complete → deadlock. + /// + /// # Panics + /// + /// This is an internal method that should only be called by `with_db_mut`. + /// Multiple concurrent calls would panic when trying to take an already-taken handle. + fn take_db_handle_for_mutation(&mut self) -> StorageHandle { + std::mem::replace(&mut self.db_handle, StorageHandle::new(None)) + } + + /// Restores the database handle after a mutation operation completes. + /// + /// This should be called with the handle extracted from the database + /// after mutations are complete. It updates the session's handle to + /// reflect any changes made during the mutation. + fn restore_db_handle(&mut self, handle: StorageHandle) { + self.db_handle = handle; + } + + /// Execute a closure with mutable access to the database. + /// + /// This method implements Salsa's required protocol for mutations: + /// 1. Takes exclusive ownership of the StorageHandle (no clones exist) + /// 2. Creates a temporary Database for the operation + /// 3. Executes your closure with `&mut Database` + /// 4. Extracts and restores the updated handle + /// + /// # Example + /// + /// ```rust,ignore + /// session.with_db_mut(|db| { + /// let file = db.get_or_create_file(path); + /// file.set_revision(db).to(new_revision); // Mutation requires exclusive access + /// }); + /// ``` + /// + /// # Why This Pattern? + /// + /// This ensures that when Salsa needs to modify inputs (via setters like + /// `set_revision`), it has exclusive access. The internal `cancel_others()` + /// call will succeed because we guarantee only one handle exists. + pub fn with_db_mut(&mut self, f: F) -> R + where + F: FnOnce(&mut Database) -> R, + { + let handle = self.take_db_handle_for_mutation(); + + let storage = handle.into_storage(); + let mut db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); + + let result = f(&mut db); + + // The database may have changed during mutations, so we need + // to extract its current handle state + let new_handle = db.storage().clone().into_zalsa_handle(); + self.restore_db_handle(new_handle); + + result + } + + /// Execute a closure with read-only access to the database. + /// + /// For read-only operations, we can safely clone the `StorageHandle` + /// since Salsa allows multiple concurrent readers. This is more + /// efficient than taking exclusive ownership. + /// + /// # Example + /// + /// ```rust,ignore + /// let content = session.with_db(|db| { + /// let file = db.get_file(path)?; + /// source_text(db, file).to_string() // Read-only query + /// }); + /// ``` + pub fn with_db(&self, f: F) -> R + where + F: FnOnce(&Database) -> R, + { + // For reads, cloning is safe and efficient + let storage = self.db_handle.clone().into_storage(); + let db = Database::from_storage(storage, self.file_system.clone(), self.files.clone()); + f(&db) + } + + /// Convert a URL to a PathBuf for file operations. + /// + /// This is needed to convert between LSP URLs and file paths for + /// SourceFile creation and tracking. + pub fn url_to_path(&self, url: &Url) -> Option { + // Only handle file:// URLs + if url.scheme() != "file" { + return None; + } + + // Decode and convert to PathBuf + let path = percent_decode_str(url.path()).decode_utf8().ok()?; + + #[cfg(windows)] + let path = path.strip_prefix('/').unwrap_or(&path); + + Some(PathBuf::from(path.as_ref())) + } + + // ===== Document Lifecycle Management ===== + // These methods encapsulate the two-layer architecture coordination: + // Layer 1 (overlays) and Layer 2 (Salsa revision tracking) + + /// Handle opening a document - sets overlay and creates file. + /// + /// This method coordinates both layers: + /// - Layer 1: Stores the document content in overlays + /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) + pub fn open_document(&mut self, url: Url, document: TextDocument) { + tracing::debug!("Opening document: {}", url); + + // Layer 1: Set overlay + self.overlays.insert(url.clone(), document); + + // Layer 2: Create file if needed (starts at revision 0) + if let Some(path) = self.url_to_path(&url) { + self.with_db_mut(|db| { + let file = db.get_or_create_file(path.clone()); + tracing::debug!( + "Created/retrieved SourceFile for {}: revision {}", + path.display(), + file.revision(db) + ); + }); + } + } + + /// Handle document changes - updates overlay and bumps revision. + /// + /// This method coordinates both layers: + /// - Layer 1: Updates the document content in overlays + /// - Layer 2: Bumps the file revision to trigger Salsa invalidation + pub fn update_document(&mut self, url: Url, document: TextDocument) { + let version = document.version(); + tracing::debug!("Updating document: {} (version {})", url, version); + + // Layer 1: Update overlay + self.overlays.insert(url.clone(), document); + + // Layer 2: Bump revision to trigger invalidation + if let Some(path) = self.url_to_path(&url) { + self.notify_file_changed(path); + } + } + + /// Handle closing a document - removes overlay and bumps revision. + /// + /// This method coordinates both layers: + /// - Layer 1: Removes the overlay (falls back to disk) + /// - Layer 2: Bumps revision to trigger re-read from disk + /// + /// Returns the removed document if it existed. + pub fn close_document(&mut self, url: &Url) -> Option { + tracing::debug!("Closing document: {}", url); + + // Layer 1: Remove overlay + let removed = self.overlays.remove(url).map(|(_, doc)| { + tracing::debug!( + "Removed overlay for closed document: {} (was version {})", + url, + doc.version() + ); + doc + }); + + // Layer 2: Bump revision to trigger re-read from disk + // We keep the file alive for potential re-opening + if let Some(path) = self.url_to_path(url) { + self.notify_file_changed(path); + } + + removed + } + + /// Internal: Notify that a file's content has changed. + /// + /// This bumps the file's revision number in Salsa, which triggers + /// invalidation of any queries that depend on the file's content. + fn notify_file_changed(&mut self, path: PathBuf) { + self.with_db_mut(|db| { + // Only bump revision if file is already being tracked + // We don't create files just for notifications + if db.has_file(&path) { + let file = db.get_or_create_file(path.clone()); + let current_rev = file.revision(db); + let new_rev = current_rev + 1; + file.set_revision(db).to(new_rev); + tracing::debug!( + "Bumped revision for {}: {} -> {}", + path.display(), + current_rev, + new_rev + ); + } else { + tracing::debug!( + "File {} not tracked, skipping revision bump", + path.display() + ); + } + }); + } + + // ===== Safe Query API ===== + // These methods encapsulate all Salsa interactions, preventing the + // "mixed database instance" bug by never exposing SourceFile or Database. + + /// Get the current content of a file (from overlay or disk). + /// + /// This is the safe way to read file content through the system. + /// The file is created if it doesn't exist, and content is read + /// through the FileSystem abstraction (overlay first, then disk). + pub fn file_content(&mut self, path: PathBuf) -> String { + use djls_workspace::db::source_text; + + self.with_db_mut(|db| { + let file = db.get_or_create_file(path); + source_text(db, file).to_string() + }) + } + + /// Get the current revision of a file, if it's being tracked. + /// + /// Returns None if the file hasn't been created yet. + pub fn file_revision(&mut self, path: &Path) -> Option { + self.with_db_mut(|db| { + db.has_file(path).then(|| { + let file = db.get_or_create_file(path.to_path_buf()); + file.revision(db) + }) + }) + } + + /// Check if a file is currently being tracked in Salsa. + pub fn has_file(&mut self, path: &Path) -> bool { + self.with_db(|db| db.has_file(path)) } } impl Default for Session { fn default() -> Self { + let overlays = Arc::new(DashMap::new()); + let files = Arc::new(DashMap::new()); + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(OsFileSystem), + )); + let db_handle = Database::new(file_system.clone(), files.clone()) + .storage() + .clone() + .into_zalsa_handle(); + Self { project: None, settings: Settings::default(), - db_handle: StorageHandle::new(None), - file_system: Arc::new(StdFileSystem), - overlays: HashMap::new(), - revision: 0, + db_handle, + file_system, + files, + overlays, client_capabilities: lsp_types::ClientCapabilities::default(), } } } + +#[cfg(test)] +mod tests { + use super::*; + use djls_workspace::LanguageId; + + #[test] + fn test_session_overlay_management() { + let session = Session::default(); + + let url = Url::parse("file:///test/file.py").unwrap(); + let document = TextDocument::new("print('hello')".to_string(), 1, LanguageId::Python); + + // Initially no overlay + assert!(!session.has_overlay(&url)); + assert!(session.get_overlay(&url).is_none()); + + // Set overlay + session.set_overlay(url.clone(), document.clone()); + assert!(session.has_overlay(&url)); + + let retrieved = session.get_overlay(&url).unwrap(); + assert_eq!(retrieved.content(), document.content()); + assert_eq!(retrieved.version(), document.version()); + + // Remove overlay + let removed = session.remove_overlay(&url).unwrap(); + assert_eq!(removed.content(), document.content()); + assert!(!session.has_overlay(&url)); + } + + #[test] + fn test_session_two_layer_architecture() { + let session = Session::default(); + + // Verify we have both layers + let _filesystem = session.file_system(); // Layer 2: FileSystem bridge + let _db = session.db(); // Layer 2: Salsa database + + // Verify overlay operations work (Layer 1) + let url = Url::parse("file:///test/integration.py").unwrap(); + let document = TextDocument::new("# Layer 1 content".to_string(), 1, LanguageId::Python); + + session.set_overlay(url.clone(), document); + assert!(session.has_overlay(&url)); + + // FileSystem should now return overlay content through LspFileSystem + // (This would be tested more thoroughly in integration tests) + } + + #[test] + fn test_revision_invalidation_chain() { + use std::path::PathBuf; + + let mut session = Session::default(); + + // Create a test file path + let path = PathBuf::from("/test/template.html"); + let url = Url::parse("file:///test/template.html").unwrap(); + + // Open document with initial content + println!("**[test]** open document with initial content"); + let document = TextDocument::new( + "

Original Content

".to_string(), + 1, + LanguageId::Other, + ); + session.open_document(url.clone(), document); + + // Try to read content - this might be where it hangs + println!("**[test]** try to read content - this might be where it hangs"); + let content1 = session.file_content(path.clone()); + assert_eq!(content1, "

Original Content

"); + + // Update document with new content + println!("**[test]** Update document with new content"); + let updated_document = + TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); + session.update_document(url.clone(), updated_document); + + // Read content again (should get new overlay content due to invalidation) + println!( + "**[test]** Read content again (should get new overlay content due to invalidation)" + ); + let content2 = session.file_content(path.clone()); + assert_eq!(content2, "

Updated Content

"); + assert_ne!(content1, content2); + + // Close document (removes overlay, bumps revision) + println!("**[test]** Close document (removes overlay, bumps revision)"); + session.close_document(&url); + + // Read content again (should now read from disk, which returns empty for missing files) + println!( + "**[test]** Read content again (should now read from disk, which returns empty for missing files)" + ); + let content3 = session.file_content(path.clone()); + assert_eq!(content3, ""); // No file on disk, returns empty + } + + #[test] + fn test_with_db_mut_preserves_files() { + use std::path::PathBuf; + + let mut session = Session::default(); + + // Create multiple files + let path1 = PathBuf::from("/test/file1.py"); + let path2 = PathBuf::from("/test/file2.py"); + + // Create files through safe API + session.file_content(path1.clone()); // Creates file1 + session.file_content(path2.clone()); // Creates file2 + + // Verify files are preserved across operations + assert!(session.has_file(&path1)); + assert!(session.has_file(&path2)); + + // Files should persist even after multiple operations + let content1 = session.file_content(path1.clone()); + let content2 = session.file_content(path2.clone()); + + // Both should return empty (no disk content) + assert_eq!(content1, ""); + assert_eq!(content2, ""); + + // One more verification + assert!(session.has_file(&path1)); + assert!(session.has_file(&path2)); + } +} diff --git a/crates/djls-workspace/src/bridge.rs b/crates/djls-workspace/src/bridge.rs deleted file mode 100644 index 2da695e..0000000 --- a/crates/djls-workspace/src/bridge.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Bridge between VFS snapshots and Salsa inputs. -//! -//! The bridge module isolates Salsa input mutation behind a single, idempotent API. -//! It ensures we only touch Salsa when content or classification changes, maximizing -//! incremental performance. - -use std::collections::HashMap; -use std::sync::Arc; - -use salsa::Setter; - -use super::db::parse_template; -use super::db::template_errors; -use super::db::Database; -use super::db::SourceFile; -use super::db::TemplateAst; -use super::db::TemplateLoaderOrder; -use super::FileId; -use super::FileKind; - -/// Owner of the Salsa [`Database`] plus the handles for updating inputs. -/// -/// [`FileStore`] serves as the bridge between the VFS (with [`FileId`]s) and Salsa (with entities). -/// It maintains a mapping from [`FileId`]s to [`SourceFile`] entities and manages the global -/// [`TemplateLoaderOrder`] input. The [`FileStore`] ensures that Salsa inputs are only mutated -/// when actual changes occur, preserving incremental computation efficiency. -pub struct FileStore { - /// The Salsa DB instance - pub db: Database, - /// Map from [`FileId`] to its Salsa input entity - files: HashMap, - /// Handle to the global template loader configuration input - template_loader: Option, -} - -impl FileStore { - /// Construct an empty store and DB. - #[must_use] - pub fn new() -> Self { - Self { - db: Database::new(), - files: HashMap::new(), - template_loader: None, - } - } - - /// Create or update the global template loader order input. - /// - /// Sets the ordered list of template root directories that Django will search - /// when resolving template names. If the input already exists, it updates the - /// existing value; otherwise, it creates a new [`TemplateLoaderOrder`] input. - pub fn set_template_loader_order(&mut self, ordered_roots: Vec) { - let roots = Arc::from(ordered_roots.into_boxed_slice()); - if let Some(tl) = self.template_loader { - tl.set_roots(&mut self.db).to(roots); - } else { - self.template_loader = Some(TemplateLoaderOrder::new(&self.db, roots)); - } - } - - // TODO: This will be replaced with direct file management - // pub(crate) fn apply_vfs_snapshot(&mut self, snap: &VfsSnapshot) { - // for (id, rec) in &snap.files { - // let new_text = snap.get_text(*id).unwrap_or_else(|| Arc::::from("")); - // let new_kind = rec.meta.kind; - - // if let Some(sf) = self.files.get(id) { - // // Update if changed — avoid touching Salsa when not needed - // if sf.kind(&self.db) != new_kind { - // sf.set_kind(&mut self.db).to(new_kind); - // } - // if sf.text(&self.db).as_ref() != &*new_text { - // sf.set_text(&mut self.db).to(new_text.clone()); - // } - // } else { - // let sf = SourceFile::new(&self.db, new_kind, new_text); - // self.files.insert(*id, sf); - // } - // } - // } - - /// Get the text content of a file by its [`FileId`]. - /// - /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub(crate) fn file_text(&self, id: FileId) -> Option> { - self.files.get(&id).map(|sf| sf.text(&self.db).clone()) - } - - /// Get the file kind classification by its [`FileId`]. - /// - /// Returns `None` if the file is not tracked in the [`FileStore`]. - pub(crate) fn file_kind(&self, id: FileId) -> Option { - self.files.get(&id).map(|sf| sf.kind(&self.db)) - } - - /// Get the parsed template AST for a file by its [`FileId`]. - /// - /// This method leverages Salsa's incremental computation to cache parsed ASTs. - /// The AST is only re-parsed when the file's content changes in the VFS. - /// Returns `None` if the file is not tracked or is not a template file. - pub(crate) fn get_template_ast(&self, id: FileId) -> Option> { - let source_file = self.files.get(&id)?; - parse_template(&self.db, *source_file) - } - - /// Get template parsing errors for a file by its [`FileId`]. - /// - /// This method provides quick access to template errors without needing the full AST. - /// Useful for diagnostics and error reporting. Returns an empty slice for - /// non-template files or files not tracked in the store. - pub(crate) fn get_template_errors(&self, id: FileId) -> Arc<[String]> { - self.files - .get(&id) - .map_or_else(|| Arc::from(vec![]), |sf| template_errors(&self.db, *sf)) - } -} - -impl Default for FileStore { - fn default() -> Self { - Self::new() - } -} - -// TODO: Re-enable tests after VFS removal is complete -// #[cfg(test)] -// mod tests { diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index c542ea4..0105616 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -1,39 +1,100 @@ //! Salsa database and input entities for workspace. //! -//! This module defines the Salsa world—what can be set and tracked incrementally. -//! Inputs are kept minimal to avoid unnecessary recomputation. +//! This module implements a two-layer architecture inspired by Ruff's design pattern +//! for efficient LSP document management with Salsa incremental computation. +//! +//! # Two-Layer Architecture +//! +//! ## Layer 1: LSP Document Management (in Session) +//! - Stores overlays in `Session` using `Arc>` +//! - TextDocument contains actual content, version, language_id +//! - Changes are immediate, no Salsa invalidation on every keystroke +//! - Thread-safe via DashMap for tower-lsp's Send+Sync requirements +//! +//! ## Layer 2: Salsa Incremental Computation (in Database) +//! - Database is pure Salsa, no file content storage +//! - Files tracked via `Arc>` for O(1) lookups +//! - SourceFile inputs only have path and revision (no text) +//! - Content read lazily through FileSystem trait +//! - LspFileSystem intercepts reads, returns overlay or disk content +//! +//! # Critical Implementation Details +//! +//! ## The Revision Dependency Trick +//! The `source_text` tracked function MUST call `file.revision(db)` to create +//! the Salsa dependency chain. Without this, revision changes won't trigger +//! invalidation of dependent queries. +//! +//! ## StorageHandle Pattern (for tower-lsp) +//! - Database itself is NOT Send+Sync (due to RefCell in Salsa's Storage) +//! - StorageHandle IS Send+Sync, enabling use across threads +//! - Session stores StorageHandle, creates Database instances on-demand +//! +//! ## Why Files are in Database, Overlays in Session +//! - Files need persistent tracking across all queries (thus in Database) +//! - Overlays are LSP-specific and change frequently (thus in Session) +//! - This separation prevents Salsa invalidation cascades on every keystroke +//! - Both are accessed via Arc for thread safety and cheap cloning +//! +//! # Data Flow +//! +//! 1. **did_open/did_change** → Update overlays in Session +//! 2. **notify_file_changed()** → Bump revision, tell Salsa something changed +//! 3. **Salsa query executes** → Calls source_text() +//! 4. **source_text() calls file.revision(db)** → Creates dependency +//! 5. **source_text() calls db.read_file_content()** → Goes through FileSystem +//! 6. **LspFileSystem intercepts** → Returns overlay if exists, else disk +//! 7. **Query gets content** → Without knowing about LSP/overlays +//! +//! This design achieves: +//! - Fast overlay updates (no Salsa invalidation) +//! - Proper incremental computation (via revision tracking) +//! - Thread safety (via Arc and StorageHandle) +//! - Clean separation of concerns (LSP vs computation) +use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; #[cfg(test)] use std::sync::Mutex; use dashmap::DashMap; -use url::Url; -use crate::{FileId, FileKind}; +use crate::{FileKind, FileSystem}; + +/// Database trait that provides file system access for Salsa queries +#[salsa::db] +pub trait Db: salsa::Database { + /// Get the file system for reading files (with overlay support) + fn fs(&self) -> Option>; + + /// Read file content through the file system + /// This is the primary way Salsa queries should read files, as it + /// automatically checks overlays before falling back to disk. + fn read_file_content(&self, path: &Path) -> std::io::Result; +} /// Salsa database root for workspace /// /// The [`Database`] provides default storage and, in tests, captures Salsa events for /// reuse/diagnostics. It serves as the core incremental computation engine, tracking /// dependencies and invalidations across all inputs and derived queries. -/// -/// This database also manages the file system overlay for the workspace, -/// mapping URLs to FileIds and storing file content. +/// +/// The database integrates with the FileSystem abstraction to read files through +/// the LspFileSystem, which automatically checks overlays before falling back to disk. #[salsa::db] #[derive(Clone)] pub struct Database { storage: salsa::Storage, - - /// Map from file URL to FileId (thread-safe) - files: DashMap, - - /// Map from FileId to file content (thread-safe) - content: DashMap>, - - /// Next FileId to allocate (thread-safe counter) - next_file_id: Arc, + + /// FileSystem integration for reading files (with overlay support) + /// This allows the database to read files through LspFileSystem, which + /// automatically checks for overlays before falling back to disk files. + fs: Option>, + + /// File tracking outside of Salsa but within Database (Arc for cheap cloning). + /// This follows Ruff's pattern where files are tracked in the Database struct + /// but not as part of Salsa's storage, enabling cheap clones via Arc. + files: Arc>, // The logs are only used for testing and demonstrating reuse: #[cfg(test)] @@ -58,101 +119,148 @@ impl Default for Database { } } }))), - files: DashMap::new(), - content: DashMap::new(), - next_file_id: Arc::new(AtomicU32::new(0)), + fs: None, + files: Arc::new(DashMap::new()), logs, } } } impl Database { - /// Create a new database instance - pub fn new() -> Self { + /// Create a new database with fresh storage. + pub fn new(file_system: Arc, files: Arc>) -> Self { Self { storage: salsa::Storage::new(None), - files: DashMap::new(), - content: DashMap::new(), - next_file_id: Arc::new(AtomicU32::new(0)), + fs: Some(file_system), + files, #[cfg(test)] logs: Arc::new(Mutex::new(None)), } } - - /// Create a new database instance from a storage handle. - /// This is used by Session::db() to create databases from the StorageHandle. - pub fn from_storage(storage: salsa::Storage) -> Self { + + /// Create a database instance from an existing storage. + /// This preserves both the file system and files Arc across database operations. + pub fn from_storage( + storage: salsa::Storage, + file_system: Arc, + files: Arc>, + ) -> Self { Self { storage, - files: DashMap::new(), - content: DashMap::new(), - next_file_id: Arc::new(AtomicU32::new(0)), + fs: Some(file_system), + files, #[cfg(test)] logs: Arc::new(Mutex::new(None)), } } - - /// Add or update a file in the workspace - pub fn set_file(&mut self, url: Url, content: String, _kind: FileKind) { - let file_id = if let Some(existing_id) = self.files.get(&url) { - *existing_id + + /// Read file content through the file system + /// This is the primary way Salsa queries should read files, as it + /// automatically checks overlays before falling back to disk. + pub fn read_file_content(&self, path: &Path) -> std::io::Result { + if let Some(fs) = &self.fs { + fs.read_to_string(path) } else { - let new_id = FileId::from_raw(self.next_file_id.fetch_add(1, Ordering::SeqCst)); - self.files.insert(url.clone(), new_id); - new_id - }; - - let content = Arc::::from(content); - self.content.insert(file_id, content.clone()); - - // TODO: Update Salsa inputs here when we connect them - } - - /// Remove a file from the workspace - pub fn remove_file(&mut self, url: &Url) { - if let Some((_, file_id)) = self.files.remove(url) { - self.content.remove(&file_id); - // TODO: Remove from Salsa when we connect inputs + std::fs::read_to_string(path) } } - - /// Get the content of a file by URL - pub fn get_file_content(&self, url: &Url) -> Option> { - let file_id = self.files.get(url)?; - self.content.get(&*file_id).map(|content| content.clone()) + + /// Get or create a SourceFile for the given path. + /// + /// This method implements Ruff's pattern for lazy file creation. Files are created + /// with an initial revision of 0 and tracked in the Database's DashMap. The Arc + /// ensures cheap cloning while maintaining thread safety. + pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { + if let Some(file_ref) = self.files.get(&path) { + // Copy the value (SourceFile is Copy) and drop the guard immediately + let file = *file_ref; + drop(file_ref); // Explicitly drop the guard to release the lock + return file; + } + + // File doesn't exist, so we need to create it + let kind = FileKind::from_path(&path); + let file = SourceFile::new(self, kind, Arc::from(path.to_string_lossy().as_ref()), 0); + + self.files.insert(path.clone(), file); + file } - - /// Get the content of a file by FileId - pub(crate) fn get_content_by_id(&self, file_id: FileId) -> Option> { - self.content.get(&file_id).map(|content| content.clone()) + + /// Check if a file is being tracked without creating it. + /// + /// This is primarily used for testing to verify that files have been + /// created without affecting the database state. + pub fn has_file(&self, path: &Path) -> bool { + self.files.contains_key(path) } - - /// Check if a file exists in the workspace - pub fn has_file(&self, url: &Url) -> bool { - self.files.contains_key(url) + + /// Get a reference to the storage for handle extraction. + /// + /// This is used by Session to extract the StorageHandle after mutations. + pub fn storage(&self) -> &salsa::Storage { + &self.storage } - - /// Get all file URLs in the workspace - pub fn files(&self) -> impl Iterator + use<'_> { - self.files.iter().map(|entry| entry.key().clone()) + + /// Consume the database and return its storage. + /// + /// This is used when you need to take ownership of the storage. + pub fn into_storage(self) -> salsa::Storage { + self.storage } } #[salsa::db] impl salsa::Database for Database {} -/// Represents a single file's classification and current content. +#[salsa::db] +impl Db for Database { + fn fs(&self) -> Option> { + self.fs.clone() + } + + fn read_file_content(&self, path: &Path) -> std::io::Result { + match &self.fs { + Some(fs) => fs.read_to_string(path), + None => std::fs::read_to_string(path), // Fallback to direct disk access + } + } +} + +/// Represents a single file without storing its content. /// -/// [`SourceFile`] is a Salsa input entity that tracks both the file's type (for routing -/// to appropriate analyzers) and its current text content. The text is stored as -/// `Arc` for efficient sharing across the incremental computation graph. +/// [`SourceFile`] is a Salsa input entity that tracks a file's path, revision, and +/// classification for analysis routing. Following Ruff's pattern, content is NOT +/// stored here but read on-demand through the `source_text` tracked function. #[salsa::input] pub struct SourceFile { /// The file's classification for analysis routing pub kind: FileKind, - /// The current text content of the file + /// The file path #[returns(ref)] - pub text: Arc, + pub path: Arc, + /// The revision number for invalidation tracking + pub revision: u64, +} + +/// Read file content through the FileSystem, creating proper Salsa dependencies. +/// +/// This is the CRITICAL function that implements Ruff's two-layer architecture. +/// The call to `file.revision(db)` creates a Salsa dependency, ensuring that +/// when the revision changes, this function (and all dependent queries) are +/// invalidated and re-executed. +#[salsa::tracked] +pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { + // This line creates the Salsa dependency on revision! Without this call, + // revision changes won't trigger invalidation + let _ = file.revision(db); + + let path = Path::new(file.path(db).as_ref()); + match db.read_file_content(path) { + Ok(content) => Arc::from(content), + Err(_) => { + Arc::from("") // Return empty string for missing files + } + } } /// Global input configuring ordered template loader roots. @@ -167,6 +275,18 @@ pub struct TemplateLoaderOrder { pub roots: Arc<[String]>, } +/// Represents a file path for Salsa tracking. +/// +/// [`FilePath`] is a Salsa input entity that tracks a file path for use in +/// path-based queries. This allows Salsa to properly track dependencies +/// on files identified by path rather than by SourceFile input. +#[salsa::input] +pub struct FilePath { + /// The file path as a string + #[returns(ref)] + pub path: Arc, +} + /// Container for a parsed Django template AST. /// /// [`TemplateAst`] wraps the parsed AST from djls-templates along with any parsing errors. @@ -183,18 +303,18 @@ pub struct TemplateAst { /// Parse a Django template file into an AST. /// /// This Salsa tracked function parses template files on-demand and caches the results. -/// The parse is only re-executed when the file's text content changes, enabling -/// efficient incremental template analysis. +/// The parse is only re-executed when the file's content changes (detected via content changes). /// /// Returns `None` for non-template files. #[salsa::tracked] -pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option> { +pub fn parse_template(db: &dyn Db, file: SourceFile) -> Option> { // Only parse template files if file.kind(db) != FileKind::Template { return None; } - let text = file.text(db); + let text_arc = source_text(db, file); + let text = text_arc.as_ref(); // Call the pure parsing function from djls-templates match djls_templates::parse_template(text) { @@ -216,6 +336,54 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option Option> { + // Read file content through the FileSystem (checks overlays first) + let path = Path::new(file_path.path(db).as_ref()); + let Ok(text) = db.read_file_content(path) else { + return None; + }; + + // Call the pure parsing function from djls-templates + match djls_templates::parse_template(&text) { + Ok((ast, errors)) => { + // Convert errors to strings + let error_strings = errors.into_iter().map(|e| e.to_string()).collect(); + Some(Arc::new(TemplateAst { + ast, + errors: error_strings, + })) + } + Err(err) => { + // Even on fatal errors, return an empty AST with the error + Some(Arc::new(TemplateAst { + ast: djls_templates::Ast::default(), + errors: vec![err.to_string()], + })) + } + } +} + +/// Get template parsing errors for a file by path. +/// +/// This Salsa tracked function extracts just the errors from the parsed template, +/// useful for diagnostics without needing the full AST. +/// +/// Reads files through the FileSystem for overlay support. +/// +/// Returns an empty vector for non-template files. +#[salsa::tracked] +pub fn template_errors_by_path(db: &dyn Db, file_path: FilePath) -> Arc<[String]> { + parse_template_by_path(db, file_path) + .map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) +} + /// Get template parsing errors for a file. /// /// This Salsa tracked function extracts just the errors from the parsed template, @@ -223,91 +391,6 @@ pub fn parse_template(db: &dyn salsa::Database, file: SourceFile) -> Option Arc<[String]> { +pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } - -#[cfg(test)] -mod tests { - use salsa::Setter; - - use super::*; - - #[test] - fn test_template_parsing_caches_result() { - let db = Database::default(); - - // Create a template file - let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }}{% endif %}"); - let file = SourceFile::new(&db, FileKind::Template, template_content.clone()); - - // First parse - should execute the parsing - let ast1 = parse_template(&db, file); - assert!(ast1.is_some()); - - // Second parse - should return cached result (same Arc) - let ast2 = parse_template(&db, file); - assert!(ast2.is_some()); - - // Verify they're the same Arc (cached) - assert!(Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } - - #[test] - fn test_template_parsing_invalidates_on_change() { - let mut db = Database::default(); - - // Create a template file - let template_content1: Arc = Arc::from("{% if user %}Hello{% endif %}"); - let file = SourceFile::new(&db, FileKind::Template, template_content1); - - // First parse - let ast1 = parse_template(&db, file); - assert!(ast1.is_some()); - - // Change the content - let template_content2: Arc = - Arc::from("{% for item in items %}{{ item }}{% endfor %}"); - file.set_text(&mut db).to(template_content2); - - // Parse again - should re-execute due to changed content - let ast2 = parse_template(&db, file); - assert!(ast2.is_some()); - - // Verify they're different Arcs (re-parsed) - assert!(!Arc::ptr_eq(&ast1.unwrap(), &ast2.unwrap())); - } - - #[test] - fn test_non_template_files_return_none() { - let db = Database::default(); - - // Create a Python file - let python_content: Arc = Arc::from("def hello():\n print('Hello')"); - let file = SourceFile::new(&db, FileKind::Python, python_content); - - // Should return None for non-template files - let ast = parse_template(&db, file); - assert!(ast.is_none()); - - // Errors should be empty for non-template files - let errors = template_errors(&db, file); - assert!(errors.is_empty()); - } - - #[test] - fn test_template_errors_tracked_separately() { - let db = Database::default(); - - // Create a template with an error (unclosed tag) - let template_content: Arc = Arc::from("{% if user %}Hello {{ user.name }"); - let file = SourceFile::new(&db, FileKind::Template, template_content); - - // Get errors - let errors1 = template_errors(&db, file); - let errors2 = template_errors(&db, file); - - // Should be cached (same Arc) - assert!(Arc::ptr_eq(&errors1, &errors2)); - } -} diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs new file mode 100644 index 0000000..c1447af --- /dev/null +++ b/crates/djls-workspace/src/document.rs @@ -0,0 +1,216 @@ +use crate::language::LanguageId; +use crate::template::ClosingBrace; +use crate::template::TemplateTagContext; +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; + +#[derive(Clone, Debug)] +pub struct TextDocument { + /// The document's content + content: String, + /// The version number of this document (from LSP) + version: i32, + /// The language identifier (python, htmldjango, etc.) + language_id: LanguageId, + /// Line index for efficient position lookups + line_index: LineIndex, +} + +impl TextDocument { + /// Create a new TextDocument with the given content + pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { + let line_index = LineIndex::new(&content); + Self { + content, + version, + language_id, + line_index, + } + } + + /// Get the document's content + pub fn content(&self) -> &str { + &self.content + } + + /// Get the version number + pub fn version(&self) -> i32 { + self.version + } + + /// Get the language identifier + pub fn language_id(&self) -> LanguageId { + self.language_id.clone() + } + + pub fn line_index(&self) -> &LineIndex { + &self.line_index + } + + pub fn get_line(&self, line: u32) -> Option { + let line_start = *self.line_index.line_starts.get(line as usize)?; + let line_end = self + .line_index + .line_starts + .get(line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + Some(self.content[line_start as usize..line_end as usize].to_string()) + } + + pub fn get_text_range(&self, range: Range) -> Option { + let start_offset = self.line_index.offset(range.start)? as usize; + let end_offset = self.line_index.offset(range.end)? as usize; + + Some(self.content[start_offset..end_offset].to_string()) + } + + /// Update the document content with LSP text changes + pub fn update( + &mut self, + changes: Vec, + version: i32, + ) { + // For now, we'll just handle full document updates + // TODO: Handle incremental updates + for change in changes { + // TextDocumentContentChangeEvent has a `text` field that's a String, not Option + self.content = change.text; + self.line_index = LineIndex::new(&self.content); + } + self.version = version; + } + + pub fn get_template_tag_context(&self, position: Position) -> Option { + let start = self.line_index.line_starts.get(position.line as usize)?; + let end = self + .line_index + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.line_index.length); + + let line = &self.content[*start as usize..end as usize]; + let char_pos: usize = position.character.try_into().ok()?; + let prefix = &line[..char_pos]; + let rest_of_line = &line[char_pos..]; + let rest_trimmed = rest_of_line.trim_start(); + + prefix.rfind("{%").map(|tag_start| { + // Check if we're immediately after {% with no space + let needs_leading_space = prefix.ends_with("{%"); + + let closing_brace = if rest_trimmed.starts_with("%}") { + ClosingBrace::FullClose + } else if rest_trimmed.starts_with('}') { + ClosingBrace::PartialClose + } else { + ClosingBrace::None + }; + + TemplateTagContext { + partial_tag: prefix[tag_start + 2..].trim().to_string(), + needs_leading_space, + closing_brace, + } + }) + } + + pub fn position_to_offset(&self, position: Position) -> Option { + self.line_index.offset(position) + } + + pub fn offset_to_position(&self, offset: u32) -> Position { + self.line_index.position(offset) + } +} + +#[derive(Clone, Debug)] +pub struct LineIndex { + pub line_starts: Vec, + pub line_starts_utf16: Vec, + pub length: u32, + pub length_utf16: u32, +} + +impl LineIndex { + #[must_use] + pub fn new(text: &str) -> Self { + let mut line_starts = vec![0]; + let mut line_starts_utf16 = vec![0]; + let mut pos_utf8 = 0; + let mut pos_utf16 = 0; + + for c in text.chars() { + pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); + pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); + if c == '\n' { + line_starts.push(pos_utf8); + line_starts_utf16.push(pos_utf16); + } + } + + Self { + line_starts, + line_starts_utf16, + length: pos_utf8, + length_utf16: pos_utf16, + } + } + + #[must_use] + pub fn offset(&self, position: Position) -> Option { + let line_start = self.line_starts.get(position.line as usize)?; + + Some(line_start + position.character) + } + + /// Convert UTF-16 LSP position to UTF-8 byte offset + pub fn offset_utf16(&self, position: Position, text: &str) -> Option { + let line_start_utf8 = self.line_starts.get(position.line as usize)?; + let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; + + // If position is at start of line, return UTF-8 line start + if position.character == 0 { + return Some(*line_start_utf8); + } + + // Find the line text + let next_line_start = self + .line_starts + .get(position.line as usize + 1) + .copied() + .unwrap_or(self.length); + + let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; + + // Convert UTF-16 character offset to UTF-8 byte offset within the line + let mut utf16_pos = 0; + let mut utf8_pos = 0; + + for c in line_text.chars() { + if utf16_pos >= position.character { + break; + } + utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); + utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); + } + + Some(line_start_utf8 + utf8_pos) + } + + #[allow(dead_code)] + #[must_use] + pub fn position(&self, offset: u32) -> Position { + let line = match self.line_starts.binary_search(&offset) { + Ok(line) => line, + Err(line) => line - 1, + }; + + let line_start = self.line_starts[line]; + let character = offset - line_start; + + Position::new(u32::try_from(line).unwrap_or(0), character) + } +} diff --git a/crates/djls-workspace/src/document/line_index.rs b/crates/djls-workspace/src/document/line_index.rs deleted file mode 100644 index cbbf645..0000000 --- a/crates/djls-workspace/src/document/line_index.rs +++ /dev/null @@ -1,90 +0,0 @@ -use tower_lsp_server::lsp_types::Position; - -#[derive(Clone, Debug)] -pub struct LineIndex { - pub line_starts: Vec, - pub line_starts_utf16: Vec, - pub length: u32, - pub length_utf16: u32, -} - -impl LineIndex { - #[must_use] - pub fn new(text: &str) -> Self { - let mut line_starts = vec![0]; - let mut line_starts_utf16 = vec![0]; - let mut pos_utf8 = 0; - let mut pos_utf16 = 0; - - for c in text.chars() { - pos_utf8 += u32::try_from(c.len_utf8()).unwrap_or(0); - pos_utf16 += u32::try_from(c.len_utf16()).unwrap_or(0); - if c == '\n' { - line_starts.push(pos_utf8); - line_starts_utf16.push(pos_utf16); - } - } - - Self { - line_starts, - line_starts_utf16, - length: pos_utf8, - length_utf16: pos_utf16, - } - } - - #[must_use] - pub fn offset(&self, position: Position) -> Option { - let line_start = self.line_starts.get(position.line as usize)?; - - Some(line_start + position.character) - } - - /// Convert UTF-16 LSP position to UTF-8 byte offset - pub fn offset_utf16(&self, position: Position, text: &str) -> Option { - let line_start_utf8 = self.line_starts.get(position.line as usize)?; - let _line_start_utf16 = self.line_starts_utf16.get(position.line as usize)?; - - // If position is at start of line, return UTF-8 line start - if position.character == 0 { - return Some(*line_start_utf8); - } - - // Find the line text - let next_line_start = self - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.length); - - let line_text = text.get(*line_start_utf8 as usize..next_line_start as usize)?; - - // Convert UTF-16 character offset to UTF-8 byte offset within the line - let mut utf16_pos = 0; - let mut utf8_pos = 0; - - for c in line_text.chars() { - if utf16_pos >= position.character { - break; - } - utf16_pos += u32::try_from(c.len_utf16()).unwrap_or(0); - utf8_pos += u32::try_from(c.len_utf8()).unwrap_or(0); - } - - Some(line_start_utf8 + utf8_pos) - } - - #[allow(dead_code)] - #[must_use] - pub fn position(&self, offset: u32) -> Position { - let line = match self.line_starts.binary_search(&offset) { - Ok(line) => line, - Err(line) => line - 1, - }; - - let line_start = self.line_starts[line]; - let character = offset - line_start; - - Position::new(u32::try_from(line).unwrap_or(0), character) - } -} diff --git a/crates/djls-workspace/src/document/mod.rs b/crates/djls-workspace/src/document/mod.rs deleted file mode 100644 index 93d443f..0000000 --- a/crates/djls-workspace/src/document/mod.rs +++ /dev/null @@ -1,130 +0,0 @@ -mod language; -mod line_index; -mod template; - -pub use language::LanguageId; -pub use line_index::LineIndex; -pub use template::ClosingBrace; -pub use template::TemplateTagContext; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; - -use crate::FileId; - -#[derive(Clone, Debug)] -pub struct TextDocument { - pub uri: String, - pub version: i32, - pub language_id: LanguageId, - pub(crate) file_id: FileId, - line_index: LineIndex, -} - -impl TextDocument { - pub(crate) fn new( - uri: String, - version: i32, - language_id: LanguageId, - file_id: FileId, - content: &str, - ) -> Self { - let line_index = LineIndex::new(content); - Self { - uri, - version, - language_id, - file_id, - line_index, - } - } - - pub(crate) fn file_id(&self) -> FileId { - self.file_id - } - - pub fn line_index(&self) -> &LineIndex { - &self.line_index - } - - pub fn get_content<'a>(&self, content: &'a str) -> &'a str { - content - } - - pub fn get_line(&self, content: &str, line: u32) -> Option { - let line_start = *self.line_index.line_starts.get(line as usize)?; - let line_end = self - .line_index - .line_starts - .get(line as usize + 1) - .copied() - .unwrap_or(self.line_index.length); - - Some(content[line_start as usize..line_end as usize].to_string()) - } - - pub fn get_text_range(&self, content: &str, range: Range) -> Option { - let start_offset = self.line_index.offset(range.start)? as usize; - let end_offset = self.line_index.offset(range.end)? as usize; - - Some(content[start_offset..end_offset].to_string()) - } - - pub fn get_template_tag_context( - &self, - content: &str, - position: Position, - ) -> Option { - let start = self.line_index.line_starts.get(position.line as usize)?; - let end = self - .line_index - .line_starts - .get(position.line as usize + 1) - .copied() - .unwrap_or(self.line_index.length); - - let line = &content[*start as usize..end as usize]; - let char_pos: usize = position.character.try_into().ok()?; - let prefix = &line[..char_pos]; - let rest_of_line = &line[char_pos..]; - let rest_trimmed = rest_of_line.trim_start(); - - prefix.rfind("{%").map(|tag_start| { - // Check if we're immediately after {% with no space - let needs_leading_space = prefix.ends_with("{%"); - - let closing_brace = if rest_trimmed.starts_with("%}") { - ClosingBrace::FullClose - } else if rest_trimmed.starts_with('}') { - ClosingBrace::PartialClose - } else { - ClosingBrace::None - }; - - TemplateTagContext { - partial_tag: prefix[tag_start + 2..].trim().to_string(), - closing_brace, - needs_leading_space, - } - }) - } - - pub fn position_to_offset(&self, position: Position) -> Option { - self.line_index.offset(position) - } - - pub fn offset_to_position(&self, offset: u32) -> Position { - self.line_index.position(offset) - } - - pub fn update_content(&mut self, content: &str) { - self.line_index = LineIndex::new(content); - } - - pub fn version(&self) -> i32 { - self.version - } - - pub fn language_id(&self) -> LanguageId { - self.language_id.clone() - } -} diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs new file mode 100644 index 0000000..26603d4 --- /dev/null +++ b/crates/djls-workspace/src/fs.rs @@ -0,0 +1,269 @@ +//! File system abstraction following Ruff's pattern +//! +//! This module provides the `FileSystem` trait that abstracts file I/O operations. +//! This allows the LSP to work with both real files and in-memory overlays. + +use dashmap::DashMap; +use std::io; +use std::path::Path; +use std::sync::Arc; +use url::Url; + +use crate::document::TextDocument; + +/// Trait for file system operations +/// +/// This follows Ruff's pattern of abstracting file system operations behind a trait, +/// allowing different implementations for testing, in-memory operation, and real file access. +pub trait FileSystem: Send + Sync { + /// Read the entire contents of a file + fn read_to_string(&self, path: &Path) -> io::Result; + + /// Check if a path exists + fn exists(&self, path: &Path) -> bool; + + /// Check if a path is a file + fn is_file(&self, path: &Path) -> bool; + + /// Check if a path is a directory + fn is_directory(&self, path: &Path) -> bool; + + /// List directory contents + fn read_directory(&self, path: &Path) -> io::Result>; + + /// Get file metadata (size, modified time, etc.) + fn metadata(&self, path: &Path) -> io::Result; +} + +/// Standard file system implementation that uses `std::fs` +pub struct OsFileSystem; + +impl FileSystem for OsFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + std::fs::read_to_string(path) + } + + fn exists(&self, path: &Path) -> bool { + path.exists() + } + + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn is_directory(&self, path: &Path) -> bool { + path.is_dir() + } + + fn read_directory(&self, path: &Path) -> io::Result> { + std::fs::read_dir(path)? + .map(|entry| entry.map(|e| e.path())) + .collect() + } + + fn metadata(&self, path: &Path) -> io::Result { + std::fs::metadata(path) + } +} + +/// LSP file system that intercepts reads for overlay files +/// +/// This implements Ruff's two-layer architecture where Layer 1 (LSP overlays) +/// takes precedence over Layer 2 (Salsa database). When a file is read, +/// this system first checks for an overlay (in-memory changes) and returns +/// that content. If no overlay exists, it falls back to reading from disk. +pub struct WorkspaceFileSystem { + /// In-memory overlays that take precedence over disk files + /// Maps URL to `TextDocument` containing current content + buffers: Arc>, + /// Fallback file system for disk operations + disk: Arc, +} + +impl WorkspaceFileSystem { + /// Create a new [`LspFileSystem`] with the given overlay storage and fallback + pub fn new(buffers: Arc>, disk: Arc) -> Self { + Self { buffers, disk } + } +} + +impl FileSystem for WorkspaceFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + if let Some(document) = path_to_url(path).and_then(|url| self.buffers.get(&url)) { + Ok(document.content().to_string()) + } else { + self.disk.read_to_string(path) + } + } + + fn exists(&self, path: &Path) -> bool { + path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + || self.disk.exists(path) + } + + fn is_file(&self, path: &Path) -> bool { + path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + || self.disk.is_file(path) + } + + fn is_directory(&self, path: &Path) -> bool { + // Overlays are never directories, so just delegate + self.disk.is_directory(path) + } + + fn read_directory(&self, path: &Path) -> io::Result> { + // Overlays are never directories, so just delegate + self.disk.read_directory(path) + } + + fn metadata(&self, path: &Path) -> io::Result { + // For overlays, we could synthesize metadata, but for simplicity, + // fall back to disk. This might need refinement for edge cases. + self.disk.metadata(path) + } +} + +/// Convert a file path to URL for overlay lookup +/// +/// This is a simplified conversion - in a full implementation, +/// you might want more robust path-to-URL conversion +fn path_to_url(path: &Path) -> Option { + if let Ok(absolute_path) = std::fs::canonicalize(path) { + return Url::from_file_path(absolute_path).ok(); + } + + // For test scenarios where the file doesn't exist on disk, + // try to create URL from the path directly if it's absolute + if path.is_absolute() { + return Url::from_file_path(path).ok(); + } + + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::TextDocument; + use crate::language::LanguageId; + + /// In-memory file system for testing + pub struct InMemoryFileSystem { + files: std::collections::HashMap, + } + + impl InMemoryFileSystem { + pub fn new() -> Self { + Self { + files: std::collections::HashMap::new(), + } + } + + pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { + self.files.insert(path, content); + } + } + + impl FileSystem for InMemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + // Simplified for testing - no directories in memory filesystem + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + // Simplified for testing + Ok(Vec::new()) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Metadata not supported in memory filesystem", + )) + } + } + + #[test] + fn test_lsp_filesystem_overlay_precedence() { + // Create a memory filesystem with some content + let mut memory_fs = InMemoryFileSystem::new(); + memory_fs.add_file( + std::path::PathBuf::from("/test/file.py"), + "original content".to_string(), + ); + + // Create overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create LspFileSystem with memory fallback + let lsp_fs = WorkspaceFileSystem::new(overlays.clone(), Arc::new(memory_fs)); + + // Before adding overlay, should read from fallback + let path = std::path::Path::new("/test/file.py"); + assert_eq!(lsp_fs.read_to_string(path).unwrap(), "original content"); + + // Add overlay - this simulates having an open document with changes + let url = Url::from_file_path("/test/file.py").unwrap(); + let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); + overlays.insert(url, document); + + // Now should read from overlay + assert_eq!(lsp_fs.read_to_string(path).unwrap(), "overlay content"); + } + + #[test] + fn test_lsp_filesystem_fallback_when_no_overlay() { + // Create memory filesystem + let mut memory_fs = InMemoryFileSystem::new(); + memory_fs.add_file( + std::path::PathBuf::from("/test/file.py"), + "disk content".to_string(), + ); + + // Create empty overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create LspFileSystem + let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + + // Should fall back to disk when no overlay exists + let path = std::path::Path::new("/test/file.py"); + assert_eq!(lsp_fs.read_to_string(path).unwrap(), "disk content"); + } + + #[test] + fn test_lsp_filesystem_other_operations_delegate() { + // Create memory filesystem + let mut memory_fs = InMemoryFileSystem::new(); + memory_fs.add_file( + std::path::PathBuf::from("/test/file.py"), + "content".to_string(), + ); + + // Create LspFileSystem + let overlays = Arc::new(DashMap::new()); + let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + + let path = std::path::Path::new("/test/file.py"); + + // These should delegate to the fallback filesystem + assert!(lsp_fs.exists(path)); + assert!(lsp_fs.is_file(path)); + assert!(!lsp_fs.is_directory(path)); + } +} diff --git a/crates/djls-workspace/src/document/language.rs b/crates/djls-workspace/src/language.rs similarity index 79% rename from crates/djls-workspace/src/document/language.rs rename to crates/djls-workspace/src/language.rs index 65c322a..8db778f 100644 --- a/crates/djls-workspace/src/document/language.rs +++ b/crates/djls-workspace/src/language.rs @@ -2,8 +2,10 @@ use crate::FileKind; #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { + Html, HtmlDjango, Other, + PlainText, Python, } @@ -11,6 +13,8 @@ impl From<&str> for LanguageId { fn from(language_id: &str) -> Self { match language_id { "django-html" | "htmldjango" => Self::HtmlDjango, + "html" => Self::Html, + "plaintext" => Self::PlainText, "python" => Self::Python, _ => Self::Other, } @@ -28,7 +32,7 @@ impl From for FileKind { match language_id { LanguageId::Python => Self::Python, LanguageId::HtmlDjango => Self::Template, - LanguageId::Other => Self::Other, + LanguageId::Html | LanguageId::PlainText | LanguageId::Other => Self::Other, } } } diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 9fbb34f..22e0faf 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,25 +1,13 @@ -mod bridge; pub mod db; mod document; -mod lsp_system; -mod system; +mod fs; +mod language; +mod template; pub use db::Database; -pub use document::{TextDocument, LanguageId}; -pub use system::{FileSystem, StdFileSystem}; - -/// File classification for routing to analyzers. -/// -/// [`FileKind`] determines how a file should be processed by downstream analyzers. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] -pub enum FileKind { - /// Python source file - Python, - /// Django template file - Template, - /// Other file type - Other, -} +pub use document::TextDocument; +pub use fs::{FileSystem, OsFileSystem, WorkspaceFileSystem}; +pub use language::LanguageId; /// Stable, compact identifier for files across the subsystem. /// @@ -43,3 +31,28 @@ impl FileId { self.0 } } + +/// File classification for routing to analyzers. +/// +/// [`FileKind`] determines how a file should be processed by downstream analyzers. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +pub enum FileKind { + /// Python source file + Python, + /// Django template file + Template, + /// Other file type + Other, +} + +impl FileKind { + /// Determine `FileKind` from a file path extension. + #[must_use] + pub fn from_path(path: &std::path::Path) -> Self { + match path.extension().and_then(|s| s.to_str()) { + Some("py") => FileKind::Python, + Some("html" | "htm") => FileKind::Template, + _ => FileKind::Other, + } + } +} diff --git a/crates/djls-workspace/src/lsp_system.rs b/crates/djls-workspace/src/lsp_system.rs deleted file mode 100644 index b03c8e8..0000000 --- a/crates/djls-workspace/src/lsp_system.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! LSP-aware file system wrapper that handles overlays -//! -//! This is the KEY pattern from Ruff - the LspSystem wraps a FileSystem -//! and intercepts reads to check for overlays first. This allows unsaved -//! changes to be used without going through Salsa. - -use std::collections::HashMap; -use std::io; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use url::Url; - -use crate::system::FileSystem; - -/// LSP-aware file system that checks overlays before disk -/// -/// This is the critical piece that makes overlays work efficiently in Ruff's -/// architecture. Instead of updating Salsa for every keystroke, we intercept -/// file reads here and return overlay content when available. -pub struct LspSystem { - /// The underlying file system (usually StdFileSystem) - inner: Arc, - - /// Map of open document URLs to their overlay content - overlays: HashMap, -} - -impl LspSystem { - /// Create a new LspSystem wrapping the given file system - pub fn new(file_system: Arc) -> Self { - Self { - inner: file_system, - overlays: HashMap::new(), - } - } - - /// Set overlay content for a document - pub fn set_overlay(&mut self, url: Url, content: String) { - self.overlays.insert(url, content); - } - - /// Remove overlay content for a document - pub fn remove_overlay(&mut self, url: &Url) { - self.overlays.remove(url); - } - - /// Check if a document has an overlay - pub fn has_overlay(&self, url: &Url) -> bool { - self.overlays.contains_key(url) - } - - /// Get overlay content if it exists - pub fn get_overlay(&self, url: &Url) -> Option<&String> { - self.overlays.get(url) - } - - /// Convert a URL to a file path - fn url_to_path(url: &Url) -> Option { - if url.scheme() == "file" { - url.to_file_path().ok().or_else(|| { - // Fallback for simple conversion - Some(PathBuf::from(url.path())) - }) - } else { - None - } - } -} - -impl FileSystem for LspSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - // First check if we have an overlay for this path - // Convert path to URL for lookup - let url = Url::from_file_path(path) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "Invalid path"))?; - - if let Some(content) = self.overlays.get(&url) { - // Return overlay content instead of reading from disk - return Ok(content.clone()); - } - - // No overlay, read from underlying file system - self.inner.read_to_string(path) - } - - fn exists(&self, path: &Path) -> bool { - // Check overlays first - if let Ok(url) = Url::from_file_path(path) { - if self.overlays.contains_key(&url) { - return true; - } - } - - self.inner.exists(path) - } - - fn is_file(&self, path: &Path) -> bool { - // Overlays are always files - if let Ok(url) = Url::from_file_path(path) { - if self.overlays.contains_key(&url) { - return true; - } - } - - self.inner.is_file(path) - } - - fn is_directory(&self, path: &Path) -> bool { - // Overlays are never directories - if let Ok(url) = Url::from_file_path(path) { - if self.overlays.contains_key(&url) { - return false; - } - } - - self.inner.is_directory(path) - } - - fn read_directory(&self, path: &Path) -> io::Result> { - // Overlays don't affect directory listings - self.inner.read_directory(path) - } - - fn metadata(&self, path: &Path) -> io::Result { - // Can't provide metadata for overlays - self.inner.metadata(path) - } -} - -/// Extension trait for working with URL-based overlays -pub trait LspSystemExt { - /// Read file content by URL, checking overlays first - fn read_url(&self, url: &Url) -> io::Result; -} - -impl LspSystemExt for LspSystem { - fn read_url(&self, url: &Url) -> io::Result { - // Check overlays first - if let Some(content) = self.overlays.get(url) { - return Ok(content.clone()); - } - - // Convert URL to path and read from file system - if let Some(path_buf) = Self::url_to_path(url) { - self.inner.read_to_string(&path_buf) - } else { - Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("Cannot convert URL to path: {}", url), - )) - } - } -} \ No newline at end of file diff --git a/crates/djls-workspace/src/system.rs b/crates/djls-workspace/src/system.rs deleted file mode 100644 index 04a1b8a..0000000 --- a/crates/djls-workspace/src/system.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! File system abstraction following Ruff's pattern -//! -//! This module provides the FileSystem trait that abstracts file I/O operations. -//! This allows the LSP to work with both real files and in-memory overlays. - -use std::io; -use std::path::Path; - -/// Trait for file system operations -/// -/// This follows Ruff's pattern of abstracting file system operations behind a trait, -/// allowing different implementations for testing, in-memory operation, and real file access. -pub trait FileSystem: Send + Sync { - /// Read the entire contents of a file - fn read_to_string(&self, path: &Path) -> io::Result; - - /// Check if a path exists - fn exists(&self, path: &Path) -> bool; - - /// Check if a path is a file - fn is_file(&self, path: &Path) -> bool; - - /// Check if a path is a directory - fn is_directory(&self, path: &Path) -> bool; - - /// List directory contents - fn read_directory(&self, path: &Path) -> io::Result>; - - /// Get file metadata (size, modified time, etc.) - fn metadata(&self, path: &Path) -> io::Result; -} - -/// Standard file system implementation that uses std::fs -pub struct StdFileSystem; - -impl FileSystem for StdFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - std::fs::read_to_string(path) - } - - fn exists(&self, path: &Path) -> bool { - path.exists() - } - - fn is_file(&self, path: &Path) -> bool { - path.is_file() - } - - fn is_directory(&self, path: &Path) -> bool { - path.is_dir() - } - - fn read_directory(&self, path: &Path) -> io::Result> { - let mut entries = Vec::new(); - for entry in std::fs::read_dir(path)? { - entries.push(entry?.path()); - } - Ok(entries) - } - - fn metadata(&self, path: &Path) -> io::Result { - std::fs::metadata(path) - } -} - -/// In-memory file system for testing -#[cfg(test)] -pub struct MemoryFileSystem { - files: std::collections::HashMap, -} - -#[cfg(test)] -impl MemoryFileSystem { - pub fn new() -> Self { - Self { - files: std::collections::HashMap::new(), - } - } - - pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { - self.files.insert(path, content); - } -} - -#[cfg(test)] -impl FileSystem for MemoryFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) - } - - fn exists(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - // Simplified for testing - no directories in memory filesystem - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - // Simplified for testing - Ok(Vec::new()) - } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "Metadata not supported in memory filesystem", - )) - } -} \ No newline at end of file diff --git a/crates/djls-workspace/src/document/template.rs b/crates/djls-workspace/src/template.rs similarity index 100% rename from crates/djls-workspace/src/document/template.rs rename to crates/djls-workspace/src/template.rs diff --git a/crates/djls-workspace/src/test_db.rs b/crates/djls-workspace/src/test_db.rs deleted file mode 100644 index 92683d5..0000000 --- a/crates/djls-workspace/src/test_db.rs +++ /dev/null @@ -1,25 +0,0 @@ -//! Test module to explore Salsa thread safety - -#[cfg(test)] -mod tests { - use crate::db::Database; - use std::thread; - - #[test] - fn test_database_clone() { - let db = Database::new(); - let _db2 = db.clone(); - println!("✅ Database can be cloned"); - } - - #[test] - #[ignore] // This will fail - fn test_database_send() { - let db = Database::new(); - let db2 = db.clone(); - - thread::spawn(move || { - let _ = db2; - }).join().unwrap(); - } -} diff --git a/task_order.md b/task_order.md new file mode 100644 index 0000000..137a402 --- /dev/null +++ b/task_order.md @@ -0,0 +1,61 @@ +# Revised Task Order for Ruff Pattern Implementation + +## The Correct Architecture Understanding + +Based on Ruff expert clarification: +- **SourceFile should NOT store text content** (our current implementation is wrong) +- **File content is read on-demand** through a `source_text` tracked function +- **Overlays are never Salsa inputs**, they're read through FileSystem +- **File revision triggers invalidation**, not content changes + +## Implementation Order + +### Phase 1: Database Foundation +1. **task-129** - Complete Database FileSystem integration + - Database needs access to LspFileSystem to read files + - This enables tracked functions to read through FileSystem + +### Phase 2: Salsa Input Restructuring +2. **task-126** - Bridge Salsa queries to LspFileSystem + - Remove `text` field from SourceFile + - Add `path` and `revision` fields + - Create `source_text` tracked function + +### Phase 3: Query Updates +3. **task-95** - Update template parsing to use source_text query + - Update all queries to use `source_text(db, file)` + - Remove direct text access from SourceFile + +### Phase 4: LSP Integration +4. **task-112** - Add file revision tracking + - Bump file revision when overlays change + - This triggers Salsa invalidation + +### Phase 5: Testing +5. **task-127** - Test overlay behavior and Salsa integration + - Verify overlays work correctly + - Test invalidation behavior + +## Key Changes from Current Implementation + +Current (WRONG): +```rust +#[salsa::input] +pub struct SourceFile { + pub text: Arc, // ❌ Storing content in Salsa +} +``` + +Target (RIGHT): +```rust +#[salsa::input] +pub struct SourceFile { + pub path: PathBuf, + pub revision: u32, // ✅ Only track changes +} + +#[salsa::tracked] +pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { + // Read through FileSystem (checks overlays first) +} +``` From 21403df0ba1a50e418021efbb9e3b2a06901e233 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 28 Aug 2025 23:07:44 -0500 Subject: [PATCH 14/30] tests --- crates/djls-workspace/src/db.rs | 175 ++++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 0105616..8a9220f 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -394,3 +394,178 @@ pub fn template_errors_by_path(db: &dyn Db, file_path: FilePath) -> Arc<[String] pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::document::TextDocument; + use crate::fs::WorkspaceFileSystem; + use crate::language::LanguageId; + use dashmap::DashMap; + use salsa::Setter; + use std::collections::HashMap; + use std::io; + use url::Url; + + // Simple in-memory filesystem for testing + struct InMemoryFileSystem { + files: HashMap, + } + + impl InMemoryFileSystem { + fn new() -> Self { + Self { + files: HashMap::new(), + } + } + + fn add_file(&mut self, path: PathBuf, content: String) { + self.files.insert(path, content); + } + } + + impl FileSystem for InMemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + Ok(vec![]) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new(io::ErrorKind::Unsupported, "Not supported")) + } + } + + #[test] + fn test_parse_template_with_overlay() { + // Create a memory filesystem with initial template content + let mut memory_fs = InMemoryFileSystem::new(); + let template_path = PathBuf::from("/test/template.html"); + memory_fs.add_file( + template_path.clone(), + "{% block content %}Original{% endblock %}".to_string(), + ); + + // Create overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create WorkspaceFileSystem that checks overlays first + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(memory_fs), + )); + + // Create database with the file system + let files = Arc::new(DashMap::new()); + let mut db = Database::new(file_system, files); + + // Create a SourceFile for the template + let file = db.get_or_create_file(template_path.clone()); + + // Parse template - should get original content from disk + let ast1 = parse_template(&db, file).expect("Should parse template"); + assert!(ast1.errors.is_empty(), "Should have no errors"); + + // Add an overlay with updated content + let url = Url::from_file_path(&template_path).unwrap(); + let updated_document = TextDocument::new( + "{% block content %}Updated from overlay{% endblock %}".to_string(), + 2, + LanguageId::Other, + ); + overlays.insert(url, updated_document); + + // Bump the file revision to trigger re-parse + file.set_revision(&mut db).to(1); + + // Parse again - should now get overlay content + let ast2 = parse_template(&db, file).expect("Should parse template"); + assert!(ast2.errors.is_empty(), "Should have no errors"); + + // Verify the content changed (we can't directly check the text, + // but the AST should be different) + // The AST will have different content in the block + assert_ne!( + format!("{:?}", ast1.ast), + format!("{:?}", ast2.ast), + "AST should change when overlay is added" + ); + } + + #[test] + fn test_parse_template_invalidation_on_revision_change() { + // Create a memory filesystem + let mut memory_fs = InMemoryFileSystem::new(); + let template_path = PathBuf::from("/test/template.html"); + memory_fs.add_file( + template_path.clone(), + "{% if true %}Initial{% endif %}".to_string(), + ); + + // Create overlay storage + let overlays = Arc::new(DashMap::new()); + + // Create WorkspaceFileSystem + let file_system = Arc::new(WorkspaceFileSystem::new( + overlays.clone(), + Arc::new(memory_fs), + )); + + // Create database + let files = Arc::new(DashMap::new()); + let mut db = Database::new(file_system, files); + + // Create a SourceFile for the template + let file = db.get_or_create_file(template_path.clone()); + + // Parse template first time + let ast1 = parse_template(&db, file).expect("Should parse"); + + // Parse again without changing revision - should return same Arc (cached) + let ast2 = parse_template(&db, file).expect("Should parse"); + assert!(Arc::ptr_eq(&ast1, &ast2), "Should return cached result"); + + // Update overlay content + let url = Url::from_file_path(&template_path).unwrap(); + let updated_document = TextDocument::new( + "{% if false %}Changed{% endif %}".to_string(), + 2, + LanguageId::Other, + ); + overlays.insert(url, updated_document); + + // Bump revision to trigger invalidation + file.set_revision(&mut db).to(1); + + // Parse again - should get different result due to invalidation + let ast3 = parse_template(&db, file).expect("Should parse"); + assert!( + !Arc::ptr_eq(&ast1, &ast3), + "Should re-execute after revision change" + ); + + // Content should be different + assert_ne!( + format!("{:?}", ast1.ast), + format!("{:?}", ast3.ast), + "AST should be different after content change" + ); + } +} From 2dd779bcdac1a8e4a8a1068aa36a80ff07d4e36e Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 07:55:37 -0500 Subject: [PATCH 15/30] Fix overlay bug: Salsa wasn't re-reading from buffers when files were opened The core issue was that when a file was opened in the LSP, if it had already been read from disk, Salsa would return cached content instead of reading from the overlay system. This happened because opening a file didn't bump its revision, so Salsa had no reason to invalidate its cache. Key changes: - Created Buffers abstraction to encapsulate shared buffer storage - Fixed Session::open_document() to bump revision when file already exists - Added comprehensive integration tests to verify overlay behavior - Refactored WorkspaceFileSystem to use Buffers instead of raw DashMap This ensures that overlays always take precedence over disk content, fixing the issue where LSP edits weren't being reflected in template parsing. --- Cargo.lock | 1 + crates/djls-server/Cargo.toml | 3 + crates/djls-server/src/lib.rs | 7 +- crates/djls-server/src/session.rs | 109 +++-- crates/djls-server/tests/lsp_integration.rs | 463 ++++++++++++++++++++ crates/djls-workspace/src/buffers.rs | 68 +++ crates/djls-workspace/src/db.rs | 13 +- crates/djls-workspace/src/fs.rs | 73 +-- crates/djls-workspace/src/lib.rs | 2 + 9 files changed, 650 insertions(+), 89 deletions(-) create mode 100644 crates/djls-server/tests/lsp_integration.rs create mode 100644 crates/djls-workspace/src/buffers.rs diff --git a/Cargo.lock b/Cargo.lock index d017be4..efc1695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,7 @@ dependencies = [ "salsa", "serde", "serde_json", + "tempfile", "tokio", "tower-lsp-server", "tracing", diff --git a/crates/djls-server/Cargo.toml b/crates/djls-server/Cargo.toml index e3f27f8..7829bf0 100644 --- a/crates/djls-server/Cargo.toml +++ b/crates/djls-server/Cargo.toml @@ -31,5 +31,8 @@ url = { workspace = true } [build-dependencies] djls-dev = { workspace = true } +[dev-dependencies] +tempfile = { workspace = true } + [lints] workspace = true diff --git a/crates/djls-server/src/lib.rs b/crates/djls-server/src/lib.rs index b601c7a..ba46830 100644 --- a/crates/djls-server/src/lib.rs +++ b/crates/djls-server/src/lib.rs @@ -1,8 +1,8 @@ mod client; mod logging; mod queue; -mod server; -mod session; +pub mod server; +pub mod session; use std::io::IsTerminal; @@ -10,7 +10,8 @@ use anyhow::Result; use tower_lsp_server::LspService; use tower_lsp_server::Server; -use crate::server::DjangoLanguageServer; +pub use crate::server::DjangoLanguageServer; +pub use crate::session::Session; pub fn run() -> Result<()> { if std::io::stdin().is_terminal() { diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 6b4adbb..189a333 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -39,7 +39,7 @@ use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::{ db::{Database, SourceFile}, - FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, + Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, }; use percent_encoding::percent_decode_str; use salsa::{Setter, StorageHandle}; @@ -77,22 +77,23 @@ pub struct Session { /// LSP server settings settings: Settings, - /// Layer 1: Thread-safe overlay storage (Arc>) + /// Layer 1: Shared buffer storage for open documents /// /// This implements Ruff's two-layer architecture where Layer 1 contains - /// LSP overlays that take precedence over disk files. The overlays map - /// document URLs to TextDocuments containing current in-memory content. + /// open document buffers that take precedence over disk files. The buffers + /// are shared between Session (which manages them) and WorkspaceFileSystem + /// (which reads from them). /// /// Key properties: - /// - Thread-safe via Arc for Send+Sync requirements + /// - Thread-safe via the Buffers abstraction /// - Contains full TextDocument with content, version, and metadata /// - Never becomes Salsa inputs - only intercepted at read time - overlays: Arc>, + buffers: Buffers, - /// File system abstraction with overlay interception + /// File system abstraction with buffer interception /// - /// This LspFileSystem bridges Layer 1 (overlays) and Layer 2 (Salsa). - /// It intercepts FileSystem::read_to_string() calls to return overlay + /// This WorkspaceFileSystem bridges Layer 1 (buffers) and Layer 2 (Salsa). + /// It intercepts FileSystem::read_to_string() calls to return buffer /// content when available, falling back to disk otherwise. file_system: Arc, @@ -132,10 +133,10 @@ impl Session { (None, Settings::default()) }; - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); let files = Arc::new(DashMap::new()); let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(OsFileSystem), )); let db_handle = Database::new(file_system.clone(), files.clone()) @@ -146,7 +147,7 @@ impl Session { Self { project, settings, - overlays, + buffers, file_system, files, client_capabilities: params.capabilities.clone(), @@ -230,32 +231,32 @@ impl Session { self.file_system.clone() } - /// Set or update an overlay for the given document URL + /// Set or update a buffer for the given document URL /// /// This implements Layer 1 of Ruff's architecture - storing in-memory /// document changes that take precedence over disk content. #[allow(dead_code)] // Used in tests pub fn set_overlay(&self, url: Url, document: TextDocument) { - self.overlays.insert(url, document); + self.buffers.open(url, document); } - /// Remove an overlay for the given document URL + /// Remove a buffer for the given document URL /// /// After removal, file reads will fall back to disk content. #[allow(dead_code)] // Used in tests pub fn remove_overlay(&self, url: &Url) -> Option { - self.overlays.remove(url).map(|(_, doc)| doc) + self.buffers.close(url) } - /// Check if an overlay exists for the given URL + /// Check if a buffer exists for the given URL #[allow(dead_code)] pub fn has_overlay(&self, url: &Url) -> bool { - self.overlays.contains_key(url) + self.buffers.contains(url) } - /// Get a copy of an overlay document + /// Get a copy of a buffered document pub fn get_overlay(&self, url: &Url) -> Option { - self.overlays.get(url).map(|doc| doc.clone()) + self.buffers.get(url) } /// Takes exclusive ownership of the database handle for mutation operations. @@ -375,41 +376,61 @@ impl Session { // These methods encapsulate the two-layer architecture coordination: // Layer 1 (overlays) and Layer 2 (Salsa revision tracking) - /// Handle opening a document - sets overlay and creates file. + /// Handle opening a document - sets buffer and creates file. /// /// This method coordinates both layers: - /// - Layer 1: Stores the document content in overlays + /// - Layer 1: Stores the document content in buffers /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) pub fn open_document(&mut self, url: Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); - // Layer 1: Set overlay - self.overlays.insert(url.clone(), document); + // Layer 1: Set buffer + self.buffers.open(url.clone(), document); - // Layer 2: Create file if needed (starts at revision 0) + // Layer 2: Create file and bump revision if it already exists + // This is crucial: if the file was already read from disk, we need to + // invalidate Salsa's cache so it re-reads through the buffer system if let Some(path) = self.url_to_path(&url) { self.with_db_mut(|db| { + // Check if file already exists (was previously read from disk) + let already_exists = db.has_file(&path); let file = db.get_or_create_file(path.clone()); - tracing::debug!( - "Created/retrieved SourceFile for {}: revision {}", - path.display(), - file.revision(db) - ); + + if already_exists { + // File was already read - bump revision to invalidate cache + let current_rev = file.revision(db); + let new_rev = current_rev + 1; + file.set_revision(db).to(new_rev); + tracing::debug!( + "Bumped revision for {} on open: {} -> {}", + path.display(), + current_rev, + new_rev + ); + + } else { + // New file - starts at revision 0 + tracing::debug!( + "Created new SourceFile for {}: revision {}", + path.display(), + file.revision(db) + ); + } }); } } - /// Handle document changes - updates overlay and bumps revision. + /// Handle document changes - updates buffer and bumps revision. /// /// This method coordinates both layers: - /// - Layer 1: Updates the document content in overlays + /// - Layer 1: Updates the document content in buffers /// - Layer 2: Bumps the file revision to trigger Salsa invalidation pub fn update_document(&mut self, url: Url, document: TextDocument) { let version = document.version(); tracing::debug!("Updating document: {} (version {})", url, version); - // Layer 1: Update overlay - self.overlays.insert(url.clone(), document); + // Layer 1: Update buffer + self.buffers.update(url.clone(), document); // Layer 2: Bump revision to trigger invalidation if let Some(path) = self.url_to_path(&url) { @@ -417,25 +438,25 @@ impl Session { } } - /// Handle closing a document - removes overlay and bumps revision. + /// Handle closing a document - removes buffer and bumps revision. /// /// This method coordinates both layers: - /// - Layer 1: Removes the overlay (falls back to disk) + /// - Layer 1: Removes the buffer (falls back to disk) /// - Layer 2: Bumps revision to trigger re-read from disk /// /// Returns the removed document if it existed. pub fn close_document(&mut self, url: &Url) -> Option { tracing::debug!("Closing document: {}", url); - // Layer 1: Remove overlay - let removed = self.overlays.remove(url).map(|(_, doc)| { + // Layer 1: Remove buffer + let removed = self.buffers.close(url); + if let Some(ref doc) = removed { tracing::debug!( - "Removed overlay for closed document: {} (was version {})", + "Removed buffer for closed document: {} (was version {})", url, doc.version() ); - doc - }); + } // Layer 2: Bump revision to trigger re-read from disk // We keep the file alive for potential re-opening @@ -512,10 +533,10 @@ impl Session { impl Default for Session { fn default() -> Self { - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); let files = Arc::new(DashMap::new()); let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(OsFileSystem), )); let db_handle = Database::new(file_system.clone(), files.clone()) @@ -529,7 +550,7 @@ impl Default for Session { db_handle, file_system, files, - overlays, + buffers, client_capabilities: lsp_types::ClientCapabilities::default(), } } diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs new file mode 100644 index 0000000..5c14607 --- /dev/null +++ b/crates/djls-server/tests/lsp_integration.rs @@ -0,0 +1,463 @@ +//! Integration tests for the LSP server's overlay → revision → invalidation flow +//! +//! These tests verify the complete two-layer architecture: +//! - Layer 1: LSP overlays (in-memory document state) +//! - Layer 2: Salsa database with revision tracking +//! +//! The tests ensure that document changes properly invalidate cached queries +//! and that overlays take precedence over disk content. + +use std::path::PathBuf; +use std::sync::Arc; + +use djls_server::DjangoLanguageServer; +use tempfile::TempDir; +use tower_lsp_server::lsp_types::{ + DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, + InitializeParams, InitializedParams, TextDocumentContentChangeEvent, TextDocumentIdentifier, + TextDocumentItem, Uri, VersionedTextDocumentIdentifier, WorkspaceFolder, +}; +use tower_lsp_server::LanguageServer; +use url::Url; + +/// Test helper that manages an LSP server instance for testing +struct TestServer { + server: DjangoLanguageServer, + _temp_dir: TempDir, + workspace_root: PathBuf, +} + +impl TestServer { + /// Create a new test server with a temporary workspace + async fn new() -> Self { + // Create temporary directory for test workspace + let temp_dir = TempDir::new().expect("Failed to create temp dir"); + let workspace_root = temp_dir.path().to_path_buf(); + + // Set up logging + let (non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); + + // Create server (guard is moved into server, so we return it too) + let server = DjangoLanguageServer::new(guard); + + // Initialize the server + let workspace_folder = WorkspaceFolder { + uri: format!("file://{}", workspace_root.display()) + .parse() + .unwrap(), + name: "test_workspace".to_string(), + }; + + let init_params = InitializeParams { + workspace_folders: Some(vec![workspace_folder]), + ..Default::default() + }; + + server + .initialize(init_params) + .await + .expect("Failed to initialize"); + server.initialized(InitializedParams {}).await; + + Self { + server, + _temp_dir: temp_dir, + workspace_root, + } + } + + /// Helper to create a file path in the test workspace + fn workspace_file(&self, name: &str) -> PathBuf { + self.workspace_root.join(name) + } + + /// Helper to create a file URL in the test workspace + fn workspace_url(&self, name: &str) -> Url { + Url::from_file_path(self.workspace_file(name)).unwrap() + } + + /// Open a document in the LSP server + async fn open_document(&self, file_name: &str, content: &str, version: i32) { + let params = DidOpenTextDocumentParams { + text_document: TextDocumentItem { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + language_id: if file_name.ends_with(".html") { + "html".to_string() + } else if file_name.ends_with(".py") { + "python".to_string() + } else { + "plaintext".to_string() + }, + version, + text: content.to_string(), + }, + }; + + self.server.did_open(params).await; + } + + /// Change a document in the LSP server + async fn change_document(&self, file_name: &str, new_content: &str, version: i32) { + let params = DidChangeTextDocumentParams { + text_document: VersionedTextDocumentIdentifier { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + version, + }, + content_changes: vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: new_content.to_string(), + }], + }; + + self.server.did_change(params).await; + } + + /// Close a document in the LSP server + async fn close_document(&self, file_name: &str) { + let params = DidCloseTextDocumentParams { + text_document: TextDocumentIdentifier { + uri: self.workspace_url(file_name).to_string().parse().unwrap(), + }, + }; + + self.server.did_close(params).await; + } + + /// Get the content of a file through the session's query system + async fn get_file_content(&self, file_name: &str) -> String { + let path = self.workspace_file(file_name); + self.server + .with_session_mut(|session| session.file_content(path)) + .await + } + + /// Write a file to disk in the test workspace + fn write_file(&self, file_name: &str, content: &str) { + let path = self.workspace_file(file_name); + std::fs::write(path, content).expect("Failed to write test file"); + } + + /// Check if a file has an overlay in the session + async fn has_overlay(&self, file_name: &str) -> bool { + let url = self.workspace_url(file_name); + self.server + .with_session(|session| session.get_overlay(&url).is_some()) + .await + } + + /// Get the revision of a file + async fn get_file_revision(&self, file_name: &str) -> Option { + let path = self.workspace_file(file_name); + self.server + .with_session_mut(|session| session.file_revision(&path)) + .await + } +} + +#[tokio::test] +async fn test_full_lsp_lifecycle() { + let server = TestServer::new().await; + let file_name = "test.html"; + + // Write initial content to disk + server.write_file(file_name, "

Disk Content

"); + + // 1. Test did_open creates overlay and file + server + .open_document(file_name, "

Overlay Content

", 1) + .await; + + // Verify overlay exists + assert!(server.has_overlay(file_name).await); + + // Verify overlay content is returned (not disk content) + let content = server.get_file_content(file_name).await; + assert_eq!(content, "

Overlay Content

"); + + // Verify file was created with revision 0 + let revision = server.get_file_revision(file_name).await; + assert_eq!(revision, Some(0)); + + // 2. Test did_change updates overlay and bumps revision + server + .change_document(file_name, "

Updated Content

", 2) + .await; + + // Verify content changed + let content = server.get_file_content(file_name).await; + assert_eq!(content, "

Updated Content

"); + + // Verify revision was bumped + let revision = server.get_file_revision(file_name).await; + assert_eq!(revision, Some(1)); + + // 3. Test did_close removes overlay and bumps revision + server.close_document(file_name).await; + + // Verify overlay is removed + assert!(!server.has_overlay(file_name).await); + + // Verify content now comes from disk + let content = server.get_file_content(file_name).await; + assert_eq!(content, "

Disk Content

"); + + // Verify revision was bumped again + let revision = server.get_file_revision(file_name).await; + assert_eq!(revision, Some(2)); +} + +#[tokio::test] +async fn test_overlay_precedence() { + let server = TestServer::new().await; + let file_name = "template.html"; + + // Write content to disk + server.write_file(file_name, "{% block content %}Disk{% endblock %}"); + + // Read content before overlay - should get disk content + let content = server.get_file_content(file_name).await; + assert_eq!(content, "{% block content %}Disk{% endblock %}"); + + // Open document with different content + server + .open_document(file_name, "{% block content %}Overlay{% endblock %}", 1) + .await; + + // Verify overlay content takes precedence + let content = server.get_file_content(file_name).await; + assert_eq!(content, "{% block content %}Overlay{% endblock %}"); + + // Close document + server.close_document(file_name).await; + + // Verify we're back to disk content + let content = server.get_file_content(file_name).await; + assert_eq!(content, "{% block content %}Disk{% endblock %}"); +} + +#[tokio::test] +async fn test_template_parsing_with_overlays() { + let server = TestServer::new().await; + let file_name = "template.html"; + + // Write initial template to disk + server.write_file(file_name, "{% if true %}Original{% endif %}"); + + // Open with different template content + server + .open_document( + file_name, + "{% for item in items %}{{ item }}{% endfor %}", + 1, + ) + .await; + use djls_workspace::db::parse_template; + + // Parse template through the session + let workspace_path = server.workspace_file(file_name); + let ast = server + .server + .with_session_mut(|session| { + session.with_db_mut(|db| { + let file = db.get_or_create_file(workspace_path); + parse_template(db, file) + }) + }) + .await; + + // Verify we parsed the overlay content (for loop), not disk content (if statement) + assert!(ast.is_some()); + let ast = ast.unwrap(); + let ast_str = format!("{:?}", ast.ast); + assert!(ast_str.contains("for") || ast_str.contains("For")); + assert!(!ast_str.contains("if") && !ast_str.contains("If")); +} + +#[tokio::test] +async fn test_multiple_documents_independent() { + let server = TestServer::new().await; + + // Open multiple documents + server.open_document("file1.html", "Content 1", 1).await; + server.open_document("file2.html", "Content 2", 1).await; + server.open_document("file3.html", "Content 3", 1).await; + + // Verify all have overlays + assert!(server.has_overlay("file1.html").await); + assert!(server.has_overlay("file2.html").await); + assert!(server.has_overlay("file3.html").await); + + // Change one document + server.change_document("file2.html", "Updated 2", 2).await; + + // Verify only file2 was updated + assert_eq!(server.get_file_content("file1.html").await, "Content 1"); + assert_eq!(server.get_file_content("file2.html").await, "Updated 2"); + assert_eq!(server.get_file_content("file3.html").await, "Content 3"); + + // Verify revision changes + assert_eq!(server.get_file_revision("file1.html").await, Some(0)); + assert_eq!(server.get_file_revision("file2.html").await, Some(1)); + assert_eq!(server.get_file_revision("file3.html").await, Some(0)); +} + +#[tokio::test] +async fn test_concurrent_overlay_updates() { + let server = Arc::new(TestServer::new().await); + + // Open initial documents + for i in 0..5 { + server + .open_document(&format!("file{}.html", i), &format!("Initial {}", i), 1) + .await; + } + + // Spawn concurrent tasks to update different documents + let mut handles = vec![]; + + for i in 0..5 { + let server_clone = Arc::clone(&server); + let handle = tokio::spawn(async move { + // Each task updates its document multiple times + for version in 2..10 { + server_clone + .change_document( + &format!("file{}.html", i), + &format!("Updated {} v{}", i, version), + version, + ) + .await; + + // Small delay to encourage interleaving + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + } + }); + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await.expect("Task failed"); + } + + // Verify final state of all documents + for i in 0..5 { + let content = server.get_file_content(&format!("file{}.html", i)).await; + assert_eq!(content, format!("Updated {} v9", i)); + + // Each document should have had 8 changes (versions 2-9) + let revision = server.get_file_revision(&format!("file{}.html", i)).await; + assert_eq!(revision, Some(8)); + } +} + +#[tokio::test] +async fn test_caching_behavior() { + let server = TestServer::new().await; + + // Open three template files + server + .open_document("template1.html", "{% block a %}1{% endblock %}", 1) + .await; + server + .open_document("template2.html", "{% block b %}2{% endblock %}", 1) + .await; + server + .open_document("template3.html", "{% block c %}3{% endblock %}", 1) + .await; + + // Parse all templates once to populate cache + for i in 1..=3 { + let _ = server + .get_file_content(&format!("template{}.html", i)) + .await; + } + + // Store initial revisions + let rev1_before = server.get_file_revision("template1.html").await.unwrap(); + let rev2_before = server.get_file_revision("template2.html").await.unwrap(); + let rev3_before = server.get_file_revision("template3.html").await.unwrap(); + + // Change only template2 + server + .change_document("template2.html", "{% block b %}CHANGED{% endblock %}", 2) + .await; + + // Verify only template2's revision changed + let rev1_after = server.get_file_revision("template1.html").await.unwrap(); + let rev2_after = server.get_file_revision("template2.html").await.unwrap(); + let rev3_after = server.get_file_revision("template3.html").await.unwrap(); + + assert_eq!( + rev1_before, rev1_after, + "template1 revision should not change" + ); + assert_eq!( + rev2_before + 1, + rev2_after, + "template2 revision should increment" + ); + assert_eq!( + rev3_before, rev3_after, + "template3 revision should not change" + ); + + // Verify content + assert_eq!( + server.get_file_content("template1.html").await, + "{% block a %}1{% endblock %}" + ); + assert_eq!( + server.get_file_content("template2.html").await, + "{% block b %}CHANGED{% endblock %}" + ); + assert_eq!( + server.get_file_content("template3.html").await, + "{% block c %}3{% endblock %}" + ); +} + +#[tokio::test] +async fn test_revision_tracking_across_lifecycle() { + let server = TestServer::new().await; + let file_name = "tracked.html"; + + // Create file on disk + server.write_file(file_name, "Initial"); + + // Open document - should create file with revision 0 + server.open_document(file_name, "Opened", 1).await; + assert_eq!(server.get_file_revision(file_name).await, Some(0)); + + // Change document multiple times + for i in 2..=5 { + server + .change_document(file_name, &format!("Change {}", i), i) + .await; + assert_eq!( + server.get_file_revision(file_name).await, + Some((i - 1) as u64), + "Revision should be {} after change {}", + i - 1, + i + ); + } + + // Close document - should bump revision one more time + server.close_document(file_name).await; + assert_eq!(server.get_file_revision(file_name).await, Some(5)); + + // Re-open document - file already exists, should bump revision to invalidate cache + server.open_document(file_name, "Reopened", 10).await; + assert_eq!( + server.get_file_revision(file_name).await, + Some(6), + "Revision should bump on re-open to invalidate cache" + ); + + // Change again + server.change_document(file_name, "Final", 11).await; + assert_eq!(server.get_file_revision(file_name).await, Some(7)); +} + diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs new file mode 100644 index 0000000..702220a --- /dev/null +++ b/crates/djls-workspace/src/buffers.rs @@ -0,0 +1,68 @@ +//! Shared buffer storage for open documents +//! +//! This module provides the `Buffers` type which represents the in-memory +//! content of open files. These buffers are shared between the Session +//! (which manages document lifecycle) and the WorkspaceFileSystem (which +//! reads from them). + +use dashmap::DashMap; +use std::sync::Arc; +use url::Url; + +use crate::document::TextDocument; + +/// Shared buffer storage between Session and FileSystem +/// +/// Buffers represent the in-memory content of open files that takes +/// precedence over disk content when reading through the FileSystem. +/// This is the key abstraction that makes the sharing between Session +/// and WorkspaceFileSystem explicit and type-safe. +#[derive(Clone, Debug)] +pub struct Buffers { + inner: Arc>, +} + +impl Buffers { + /// Create a new empty buffer storage + pub fn new() -> Self { + Self { + inner: Arc::new(DashMap::new()), + } + } + + /// Open a document in the buffers + pub fn open(&self, url: Url, document: TextDocument) { + self.inner.insert(url, document); + } + + /// Update an open document + pub fn update(&self, url: Url, document: TextDocument) { + self.inner.insert(url, document); + } + + /// Close a document and return it if it was open + pub fn close(&self, url: &Url) -> Option { + self.inner.remove(url).map(|(_, doc)| doc) + } + + /// Get a document if it's open + pub fn get(&self, url: &Url) -> Option { + self.inner.get(url).map(|entry| entry.clone()) + } + + /// Check if a document is open + pub fn contains(&self, url: &Url) -> bool { + self.inner.contains_key(url) + } + + /// Iterate over all open buffers (for debugging) + pub fn iter(&self) -> impl Iterator + '_ { + self.inner.iter().map(|entry| (entry.key().clone(), entry.value().clone())) + } +} + +impl Default for Buffers { + fn default() -> Self { + Self::new() + } +} \ No newline at end of file diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 8a9220f..3fba217 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -398,6 +398,7 @@ pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { #[cfg(test)] mod tests { use super::*; + use crate::buffers::Buffers; use crate::document::TextDocument; use crate::fs::WorkspaceFileSystem; use crate::language::LanguageId; @@ -464,11 +465,11 @@ mod tests { ); // Create overlay storage - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); // Create WorkspaceFileSystem that checks overlays first let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(memory_fs), )); @@ -490,7 +491,7 @@ mod tests { 2, LanguageId::Other, ); - overlays.insert(url, updated_document); + buffers.open(url, updated_document); // Bump the file revision to trigger re-parse file.set_revision(&mut db).to(1); @@ -520,11 +521,11 @@ mod tests { ); // Create overlay storage - let overlays = Arc::new(DashMap::new()); + let buffers = Buffers::new(); // Create WorkspaceFileSystem let file_system = Arc::new(WorkspaceFileSystem::new( - overlays.clone(), + buffers.clone(), Arc::new(memory_fs), )); @@ -549,7 +550,7 @@ mod tests { 2, LanguageId::Other, ); - overlays.insert(url, updated_document); + buffers.open(url, updated_document); // Bump revision to trigger invalidation file.set_revision(&mut db).to(1); diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 26603d4..151c6d6 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -3,13 +3,12 @@ //! This module provides the `FileSystem` trait that abstracts file I/O operations. //! This allows the LSP to work with both real files and in-memory overlays. -use dashmap::DashMap; use std::io; use std::path::Path; use std::sync::Arc; use url::Url; -use crate::document::TextDocument; +use crate::buffers::Buffers; /// Trait for file system operations /// @@ -66,43 +65,43 @@ impl FileSystem for OsFileSystem { } } -/// LSP file system that intercepts reads for overlay files +/// LSP file system that intercepts reads for buffered files /// -/// This implements Ruff's two-layer architecture where Layer 1 (LSP overlays) +/// This implements Ruff's two-layer architecture where Layer 1 (open buffers) /// takes precedence over Layer 2 (Salsa database). When a file is read, -/// this system first checks for an overlay (in-memory changes) and returns -/// that content. If no overlay exists, it falls back to reading from disk. +/// this system first checks for a buffer (in-memory content) and returns +/// that content. If no buffer exists, it falls back to reading from disk. pub struct WorkspaceFileSystem { - /// In-memory overlays that take precedence over disk files - /// Maps URL to `TextDocument` containing current content - buffers: Arc>, + /// In-memory buffers that take precedence over disk files + buffers: Buffers, /// Fallback file system for disk operations disk: Arc, } impl WorkspaceFileSystem { - /// Create a new [`LspFileSystem`] with the given overlay storage and fallback - pub fn new(buffers: Arc>, disk: Arc) -> Self { + /// Create a new [`WorkspaceFileSystem`] with the given buffer storage and fallback + pub fn new(buffers: Buffers, disk: Arc) -> Self { Self { buffers, disk } } } impl FileSystem for WorkspaceFileSystem { fn read_to_string(&self, path: &Path) -> io::Result { - if let Some(document) = path_to_url(path).and_then(|url| self.buffers.get(&url)) { - Ok(document.content().to_string()) - } else { - self.disk.read_to_string(path) + if let Some(url) = path_to_url(path) { + if let Some(document) = self.buffers.get(&url) { + return Ok(document.content().to_string()); + } } + self.disk.read_to_string(path) } fn exists(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.exists(path) } fn is_file(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains_key(&url)) + path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.is_file(path) } @@ -128,22 +127,24 @@ impl FileSystem for WorkspaceFileSystem { /// This is a simplified conversion - in a full implementation, /// you might want more robust path-to-URL conversion fn path_to_url(path: &Path) -> Option { - if let Ok(absolute_path) = std::fs::canonicalize(path) { - return Url::from_file_path(absolute_path).ok(); - } - - // For test scenarios where the file doesn't exist on disk, - // try to create URL from the path directly if it's absolute + // For absolute paths, use them directly without canonicalization + // This ensures consistency with how URLs are created when storing overlays if path.is_absolute() { return Url::from_file_path(path).ok(); } + // Only try to canonicalize for relative paths + if let Ok(absolute_path) = std::fs::canonicalize(path) { + return Url::from_file_path(absolute_path).ok(); + } + None } #[cfg(test)] mod tests { use super::*; + use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; @@ -207,22 +208,22 @@ mod tests { "original content".to_string(), ); - // Create overlay storage - let overlays = Arc::new(DashMap::new()); + // Create buffer storage + let buffers = Buffers::new(); // Create LspFileSystem with memory fallback - let lsp_fs = WorkspaceFileSystem::new(overlays.clone(), Arc::new(memory_fs)); + let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); - // Before adding overlay, should read from fallback + // Before adding buffer, should read from fallback let path = std::path::Path::new("/test/file.py"); assert_eq!(lsp_fs.read_to_string(path).unwrap(), "original content"); - // Add overlay - this simulates having an open document with changes + // Add buffer - this simulates having an open document with changes let url = Url::from_file_path("/test/file.py").unwrap(); let document = TextDocument::new("overlay content".to_string(), 1, LanguageId::Python); - overlays.insert(url, document); + buffers.open(url, document); - // Now should read from overlay + // Now should read from buffer assert_eq!(lsp_fs.read_to_string(path).unwrap(), "overlay content"); } @@ -235,13 +236,13 @@ mod tests { "disk content".to_string(), ); - // Create empty overlay storage - let overlays = Arc::new(DashMap::new()); + // Create empty buffer storage + let buffers = Buffers::new(); // Create LspFileSystem - let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); - // Should fall back to disk when no overlay exists + // Should fall back to disk when no buffer exists let path = std::path::Path::new("/test/file.py"); assert_eq!(lsp_fs.read_to_string(path).unwrap(), "disk content"); } @@ -256,8 +257,8 @@ mod tests { ); // Create LspFileSystem - let overlays = Arc::new(DashMap::new()); - let lsp_fs = WorkspaceFileSystem::new(overlays, Arc::new(memory_fs)); + let buffers = Buffers::new(); + let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); let path = std::path::Path::new("/test/file.py"); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 22e0faf..a3a8d01 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,9 +1,11 @@ +mod buffers; pub mod db; mod document; mod fs; mod language; mod template; +pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; pub use fs::{FileSystem, OsFileSystem, WorkspaceFileSystem}; From f3fb8e70451e9490cf18f7c906496b14c5e2c064 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 15:35:02 -0500 Subject: [PATCH 16/30] Improve documentation and consolidate path/URL utilities - Added comprehensive module-level documentation to all djls-workspace modules - Consolidated scattered URL/path conversion utilities into paths module - Added documentation explaining the 'why' for key types and abstractions - Added #[must_use] annotations to constructors and getters - Focused on explaining architecture and design decisions rather than obvious behavior --- Cargo.lock | 1 + crates/djls-server/src/session.rs | 55 +----- crates/djls-server/tests/lsp_integration.rs | 6 +- crates/djls-workspace/Cargo.toml | 1 + crates/djls-workspace/src/buffers.rs | 11 +- crates/djls-workspace/src/db.rs | 5 +- crates/djls-workspace/src/document.rs | 22 +++ crates/djls-workspace/src/fs.rs | 30 +-- crates/djls-workspace/src/language.rs | 9 + crates/djls-workspace/src/lib.rs | 15 ++ crates/djls-workspace/src/paths.rs | 200 ++++++++++++++++++++ crates/djls-workspace/src/template.rs | 23 ++- 12 files changed, 295 insertions(+), 83 deletions(-) create mode 100644 crates/djls-workspace/src/paths.rs diff --git a/Cargo.lock b/Cargo.lock index efc1695..2722022 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,6 +513,7 @@ dependencies = [ "djls-project", "djls-templates", "notify", + "percent-encoding", "salsa", "tempfile", "tokio", diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 189a333..d0ddafa 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -39,9 +39,8 @@ use djls_conf::Settings; use djls_project::DjangoProject; use djls_workspace::{ db::{Database, SourceFile}, - Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, + paths, Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, }; -use percent_encoding::percent_decode_str; use salsa::{Setter, StorageHandle}; use tower_lsp_server::lsp_types; use url::Url; @@ -171,25 +170,7 @@ impl Session { /// Converts a `file:` URI into an absolute `PathBuf`. fn uri_to_pathbuf(uri: &lsp_types::Uri) -> Option { - // Check if the scheme is "file" - if uri.scheme().is_none_or(|s| s.as_str() != "file") { - return None; - } - - // Get the path part as a string - let encoded_path_str = uri.path().as_str(); - - // Decode the percent-encoded path string - let decoded_path_cow = percent_decode_str(encoded_path_str).decode_utf8_lossy(); - let path_str = decoded_path_cow.as_ref(); - - #[cfg(windows)] - let path_str = { - // Remove leading '/' for paths like /C:/... - path_str.strip_prefix('/').unwrap_or(path_str) - }; - - Some(PathBuf::from(path_str)) + paths::lsp_uri_to_path(uri) } pub fn project(&self) -> Option<&DjangoProject> { @@ -353,29 +334,6 @@ impl Session { f(&db) } - /// Convert a URL to a PathBuf for file operations. - /// - /// This is needed to convert between LSP URLs and file paths for - /// SourceFile creation and tracking. - pub fn url_to_path(&self, url: &Url) -> Option { - // Only handle file:// URLs - if url.scheme() != "file" { - return None; - } - - // Decode and convert to PathBuf - let path = percent_decode_str(url.path()).decode_utf8().ok()?; - - #[cfg(windows)] - let path = path.strip_prefix('/').unwrap_or(&path); - - Some(PathBuf::from(path.as_ref())) - } - - // ===== Document Lifecycle Management ===== - // These methods encapsulate the two-layer architecture coordination: - // Layer 1 (overlays) and Layer 2 (Salsa revision tracking) - /// Handle opening a document - sets buffer and creates file. /// /// This method coordinates both layers: @@ -390,12 +348,12 @@ impl Session { // Layer 2: Create file and bump revision if it already exists // This is crucial: if the file was already read from disk, we need to // invalidate Salsa's cache so it re-reads through the buffer system - if let Some(path) = self.url_to_path(&url) { + if let Some(path) = paths::url_to_path(&url) { self.with_db_mut(|db| { // Check if file already exists (was previously read from disk) let already_exists = db.has_file(&path); let file = db.get_or_create_file(path.clone()); - + if already_exists { // File was already read - bump revision to invalidate cache let current_rev = file.revision(db); @@ -407,7 +365,6 @@ impl Session { current_rev, new_rev ); - } else { // New file - starts at revision 0 tracing::debug!( @@ -433,7 +390,7 @@ impl Session { self.buffers.update(url.clone(), document); // Layer 2: Bump revision to trigger invalidation - if let Some(path) = self.url_to_path(&url) { + if let Some(path) = paths::url_to_path(&url) { self.notify_file_changed(path); } } @@ -460,7 +417,7 @@ impl Session { // Layer 2: Bump revision to trigger re-read from disk // We keep the file alive for potential re-opening - if let Some(path) = self.url_to_path(url) { + if let Some(path) = paths::url_to_path(url) { self.notify_file_changed(path); } diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index 5c14607..4bfc666 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -15,7 +15,7 @@ use tempfile::TempDir; use tower_lsp_server::lsp_types::{ DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, InitializeParams, InitializedParams, TextDocumentContentChangeEvent, TextDocumentIdentifier, - TextDocumentItem, Uri, VersionedTextDocumentIdentifier, WorkspaceFolder, + TextDocumentItem, VersionedTextDocumentIdentifier, WorkspaceFolder, }; use tower_lsp_server::LanguageServer; use url::Url; @@ -35,7 +35,7 @@ impl TestServer { let workspace_root = temp_dir.path().to_path_buf(); // Set up logging - let (non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); + let (_non_blocking, guard) = tracing_appender::non_blocking(std::io::sink()); // Create server (guard is moved into server, so we return it too) let server = DjangoLanguageServer::new(guard); @@ -73,7 +73,7 @@ impl TestServer { /// Helper to create a file URL in the test workspace fn workspace_url(&self, name: &str) -> Url { - Url::from_file_path(self.workspace_file(name)).unwrap() + djls_workspace::paths::path_to_url(&self.workspace_file(name)).unwrap() } /// Open a document in the LSP server diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index 0a46bd8..b1fa2e0 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -11,6 +11,7 @@ anyhow = { workspace = true } camino = { workspace = true } dashmap = { workspace = true } notify = { workspace = true } +percent-encoding = { workspace = true } salsa = { workspace = true } tokio = { workspace = true } tower-lsp-server = { workspace = true } diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 702220a..0d400e9 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -24,6 +24,7 @@ pub struct Buffers { impl Buffers { /// Create a new empty buffer storage + #[must_use] pub fn new() -> Self { Self { inner: Arc::new(DashMap::new()), @@ -41,23 +42,28 @@ impl Buffers { } /// Close a document and return it if it was open + #[must_use] pub fn close(&self, url: &Url) -> Option { self.inner.remove(url).map(|(_, doc)| doc) } /// Get a document if it's open + #[must_use] pub fn get(&self, url: &Url) -> Option { self.inner.get(url).map(|entry| entry.clone()) } /// Check if a document is open + #[must_use] pub fn contains(&self, url: &Url) -> bool { self.inner.contains_key(url) } /// Iterate over all open buffers (for debugging) pub fn iter(&self) -> impl Iterator + '_ { - self.inner.iter().map(|entry| (entry.key().clone(), entry.value().clone())) + self.inner + .iter() + .map(|entry| (entry.key().clone(), entry.value().clone())) } } @@ -65,4 +71,5 @@ impl Default for Buffers { fn default() -> Self { Self::new() } -} \ No newline at end of file +} + diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 3fba217..1892e2d 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -406,7 +406,6 @@ mod tests { use salsa::Setter; use std::collections::HashMap; use std::io; - use url::Url; // Simple in-memory filesystem for testing struct InMemoryFileSystem { @@ -485,7 +484,7 @@ mod tests { assert!(ast1.errors.is_empty(), "Should have no errors"); // Add an overlay with updated content - let url = Url::from_file_path(&template_path).unwrap(); + let url = crate::paths::path_to_url(&template_path).unwrap(); let updated_document = TextDocument::new( "{% block content %}Updated from overlay{% endblock %}".to_string(), 2, @@ -544,7 +543,7 @@ mod tests { assert!(Arc::ptr_eq(&ast1, &ast2), "Should return cached result"); // Update overlay content - let url = Url::from_file_path(&template_path).unwrap(); + let url = crate::paths::path_to_url(&template_path).unwrap(); let updated_document = TextDocument::new( "{% if false %}Changed{% endif %}".to_string(), 2, diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index c1447af..9def945 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -1,9 +1,21 @@ +//! LSP text document representation with efficient line indexing +//! +//! [`TextDocument`] stores open file content with version tracking for the LSP protocol. +//! Pre-computed line indices enable O(1) position lookups, which is critical for +//! performance when handling frequent position-based operations like hover, completion, +//! and diagnostics. + use crate::language::LanguageId; use crate::template::ClosingBrace; use crate::template::TemplateTagContext; use tower_lsp_server::lsp_types::Position; use tower_lsp_server::lsp_types::Range; +/// In-memory representation of an open document in the LSP. +/// +/// Combines document content with metadata needed for LSP operations, +/// including version tracking for synchronization and pre-computed line +/// indices for efficient position lookups. #[derive(Clone, Debug)] pub struct TextDocument { /// The document's content @@ -18,6 +30,7 @@ pub struct TextDocument { impl TextDocument { /// Create a new TextDocument with the given content + #[must_use] pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { let line_index = LineIndex::new(&content); Self { @@ -29,20 +42,24 @@ impl TextDocument { } /// Get the document's content + #[must_use] pub fn content(&self) -> &str { &self.content } /// Get the version number + #[must_use] pub fn version(&self) -> i32 { self.version } /// Get the language identifier + #[must_use] pub fn language_id(&self) -> LanguageId { self.language_id.clone() } + #[must_use] pub fn line_index(&self) -> &LineIndex { &self.line_index } @@ -126,6 +143,11 @@ impl TextDocument { } } +/// Pre-computed line start positions for efficient position/offset conversion. +/// +/// Computing line positions on every lookup would be O(n) where n is the document size. +/// By pre-computing during document creation/updates, we get O(1) lookups for line starts +/// and O(log n) for position-to-offset conversions via binary search. #[derive(Clone, Debug)] pub struct LineIndex { pub line_starts: Vec, diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 151c6d6..8ef755f 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -6,9 +6,8 @@ use std::io; use std::path::Path; use std::sync::Arc; -use url::Url; -use crate::buffers::Buffers; +use crate::{buffers::Buffers, paths}; /// Trait for file system operations /// @@ -80,6 +79,7 @@ pub struct WorkspaceFileSystem { impl WorkspaceFileSystem { /// Create a new [`WorkspaceFileSystem`] with the given buffer storage and fallback + #[must_use] pub fn new(buffers: Buffers, disk: Arc) -> Self { Self { buffers, disk } } @@ -87,7 +87,7 @@ impl WorkspaceFileSystem { impl FileSystem for WorkspaceFileSystem { fn read_to_string(&self, path: &Path) -> io::Result { - if let Some(url) = path_to_url(path) { + if let Some(url) = paths::path_to_url(path) { if let Some(document) = self.buffers.get(&url) { return Ok(document.content().to_string()); } @@ -96,12 +96,12 @@ impl FileSystem for WorkspaceFileSystem { } fn exists(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) + paths::path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.exists(path) } fn is_file(&self, path: &Path) -> bool { - path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) + paths::path_to_url(path).is_some_and(|url| self.buffers.contains(&url)) || self.disk.is_file(path) } @@ -122,31 +122,13 @@ impl FileSystem for WorkspaceFileSystem { } } -/// Convert a file path to URL for overlay lookup -/// -/// This is a simplified conversion - in a full implementation, -/// you might want more robust path-to-URL conversion -fn path_to_url(path: &Path) -> Option { - // For absolute paths, use them directly without canonicalization - // This ensures consistency with how URLs are created when storing overlays - if path.is_absolute() { - return Url::from_file_path(path).ok(); - } - - // Only try to canonicalize for relative paths - if let Ok(absolute_path) = std::fs::canonicalize(path) { - return Url::from_file_path(absolute_path).ok(); - } - - None -} - #[cfg(test)] mod tests { use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; + use url::Url; /// In-memory file system for testing pub struct InMemoryFileSystem { diff --git a/crates/djls-workspace/src/language.rs b/crates/djls-workspace/src/language.rs index 8db778f..f92811c 100644 --- a/crates/djls-workspace/src/language.rs +++ b/crates/djls-workspace/src/language.rs @@ -1,5 +1,14 @@ +//! Language identification for document routing +//! +//! Maps LSP language identifiers to internal [`FileKind`] for analyzer routing. +//! Language IDs come from the LSP client and determine how files are processed. + use crate::FileKind; +/// Language identifier as reported by the LSP client. +/// +/// These identifiers follow VS Code's language ID conventions and determine +/// which analyzers and features are available for a document. #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { Html, diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index a3a8d01..9e17d15 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -1,8 +1,23 @@ +//! Workspace management for the Django Language Server +//! +//! This crate provides the core workspace functionality including document management, +//! file system abstractions, and Salsa integration for incremental computation of +//! Django projects. +//! +//! # Key Components +//! +//! - [`Buffers`] - Thread-safe storage for open documents +//! - [`Database`] - Salsa database for incremental computation +//! - [`TextDocument`] - LSP document representation with efficient indexing +//! - [`FileSystem`] - Abstraction layer for file operations with overlay support +//! - [`paths`] - Consistent URL/path conversion utilities + mod buffers; pub mod db; mod document; mod fs; mod language; +pub mod paths; mod template; pub use buffers::Buffers; diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs new file mode 100644 index 0000000..515df66 --- /dev/null +++ b/crates/djls-workspace/src/paths.rs @@ -0,0 +1,200 @@ +//! Path and URL conversion utilities +//! +//! This module provides consistent conversion between file paths and URLs, +//! handling platform-specific differences and encoding issues. + +use std::path::{Path, PathBuf}; +use tower_lsp_server::lsp_types; +use url::Url; + +/// Convert a file:// URL to a `PathBuf` +/// +/// Handles percent-encoding and platform-specific path formats (e.g., Windows drives). +#[must_use] +pub fn url_to_path(url: &Url) -> Option { + // Only handle file:// URLs + if url.scheme() != "file" { + return None; + } + + // Get the path component and decode percent-encoding + let path = percent_encoding::percent_decode_str(url.path()) + .decode_utf8() + .ok()?; + + #[cfg(windows)] + let path = { + // Remove leading '/' for paths like /C:/... + path.strip_prefix('/').unwrap_or(&path) + }; + + Some(PathBuf::from(path.as_ref())) +} + +/// Convert an LSP URI to a `PathBuf` +/// +/// This is a convenience wrapper that parses the LSP URI string and converts it. +pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { + // Parse the URI string as a URL + let url = Url::parse(lsp_uri.as_str()).ok()?; + url_to_path(&url) +} + +/// Convert a Path to a file:// URL +/// +/// Handles both absolute and relative paths. Relative paths are resolved +/// to absolute paths before conversion. +#[must_use] +pub fn path_to_url(path: &Path) -> Option { + // For absolute paths, convert directly + if path.is_absolute() { + return Url::from_file_path(path).ok(); + } + + // For relative paths, try to make them absolute first + if let Ok(absolute_path) = std::fs::canonicalize(path) { + return Url::from_file_path(absolute_path).ok(); + } + + // If canonicalization fails, try converting as-is (might fail) + Url::from_file_path(path).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_url_to_path_basic() { + let url = Url::parse("file:///home/user/file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("/home/user/file.txt")); + } + + #[test] + fn test_url_to_path_with_spaces() { + let url = Url::parse("file:///home/user/my%20file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("/home/user/my file.txt")); + } + + #[test] + fn test_url_to_path_non_file_scheme() { + let url = Url::parse("https://example.com/file.txt").unwrap(); + assert!(url_to_path(&url).is_none()); + } + + #[cfg(windows)] + #[test] + fn test_url_to_path_windows() { + let url = Url::parse("file:///C:/Users/user/file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("C:/Users/user/file.txt")); + } + + #[test] + fn test_path_to_url_absolute() { + let path = if cfg!(windows) { + PathBuf::from("C:/Users/user/file.txt") + } else { + PathBuf::from("/home/user/file.txt") + }; + + let url = path_to_url(&path).unwrap(); + assert_eq!(url.scheme(), "file"); + assert!(url.path().contains("file.txt")); + } + + #[test] + fn test_round_trip() { + let original_path = if cfg!(windows) { + PathBuf::from("C:/Users/user/test file.txt") + } else { + PathBuf::from("/home/user/test file.txt") + }; + + let url = path_to_url(&original_path).unwrap(); + let converted_path = url_to_path(&url).unwrap(); + + assert_eq!(original_path, converted_path); + } + + #[test] + fn test_url_with_localhost() { + // Some systems use file://localhost/path format + let url = Url::parse("file://localhost/home/user/file.txt").unwrap(); + let path = url_to_path(&url); + + // Current implementation might not handle this correctly + // since it only checks scheme, not host + if let Some(p) = path { + assert_eq!(p, PathBuf::from("/home/user/file.txt")); + } + } + + #[test] + fn test_url_with_empty_host() { + // Standard file:///path format (three slashes, empty host) + let url = Url::parse("file:///home/user/file.txt").unwrap(); + let path = url_to_path(&url).unwrap(); + assert_eq!(path, PathBuf::from("/home/user/file.txt")); + } + + #[cfg(windows)] + #[test] + fn test_unc_path_to_url() { + // UNC paths like \\server\share\file.txt + let unc_path = PathBuf::from(r"\\server\share\file.txt"); + let url = path_to_url(&unc_path); + + // Check if UNC paths are handled + if let Some(u) = url { + // UNC paths should convert to file://server/share/file.txt + assert!(u.to_string().contains("server")); + assert!(u.to_string().contains("share")); + } + } + + #[test] + fn test_relative_path_with_dotdot() { + // Test relative paths with .. that might not exist + let path = PathBuf::from("../some/nonexistent/path.txt"); + let url = path_to_url(&path); + + // This might fail if the path doesn't exist and can't be canonicalized + // Current implementation falls back to trying direct conversion + assert!(url.is_none() || url.is_some()); + } + + #[test] + fn test_path_with_special_chars() { + // Test paths with special characters that need encoding + let path = PathBuf::from("/home/user/file with spaces & special!.txt"); + let url = path_to_url(&path).unwrap(); + + // Should be properly percent-encoded + assert!(url.as_str().contains("%20") || url.as_str().contains("with%20spaces")); + + // Round-trip should work + let back = url_to_path(&url).unwrap(); + assert_eq!(back, path); + } + + #[test] + fn test_url_with_query_or_fragment() { + // URLs with query parameters or fragments should probably be rejected + let url_with_query = Url::parse("file:///path/file.txt?query=param").unwrap(); + let url_with_fragment = Url::parse("file:///path/file.txt#section").unwrap(); + + // These should still work, extracting just the path part + let path1 = url_to_path(&url_with_query); + let path2 = url_to_path(&url_with_fragment); + + if let Some(p) = path1 { + assert_eq!(p, PathBuf::from("/path/file.txt")); + } + if let Some(p) = path2 { + assert_eq!(p, PathBuf::from("/path/file.txt")); + } + } +} diff --git a/crates/djls-workspace/src/template.rs b/crates/djls-workspace/src/template.rs index 2a0547c..2ebc324 100644 --- a/crates/djls-workspace/src/template.rs +++ b/crates/djls-workspace/src/template.rs @@ -1,13 +1,32 @@ +//! Django template context detection for completions +//! +//! Detects cursor position context within Django template tags to provide +//! appropriate completions and auto-closing behavior. + +/// Tracks what closing characters are needed to complete a template tag. +/// +/// Used to determine whether the completion system needs to insert +/// closing braces when completing a Django template tag. #[derive(Debug)] pub enum ClosingBrace { + /// No closing brace present - need to add full `%}` or `}}` None, - PartialClose, // just } - FullClose, // %} + /// Partial close present (just `}`) - need to add `%` or second `}` + PartialClose, + /// Full close present (`%}` or `}}`) - no closing needed + FullClose, } +/// Cursor context within a Django template tag for completion support. +/// +/// Captures the state around the cursor position to provide intelligent +/// completions and determine what text needs to be inserted. #[derive(Debug)] pub struct TemplateTagContext { + /// The partial tag text before the cursor (e.g., "loa" for "{% loa|") pub partial_tag: String, + /// What closing characters are already present after the cursor pub closing_brace: ClosingBrace, + /// Whether a space is needed before the completion (true if cursor is right after `{%`) pub needs_leading_space: bool, } From 89e979ba3f1afefbe5df747694ad398b4d53f9a9 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 15:39:23 -0500 Subject: [PATCH 17/30] Add cross-references and improve documentation consistency - Added [] cross-references between related types - Fixed parameter naming consistency (lsp_uri -> uri) - Added Returns sections to document when functions return None - Added example to url_to_path function - Linked Buffers <-> WorkspaceFileSystem relationship - Linked LanguageId -> FileKind conversion --- crates/djls-workspace/src/buffers.rs | 7 +++++-- crates/djls-workspace/src/fs.rs | 9 ++++++--- crates/djls-workspace/src/language.rs | 3 ++- crates/djls-workspace/src/paths.rs | 23 +++++++++++++++++++++-- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 0d400e9..ae42916 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -14,9 +14,12 @@ use crate::document::TextDocument; /// Shared buffer storage between Session and FileSystem /// /// Buffers represent the in-memory content of open files that takes -/// precedence over disk content when reading through the FileSystem. +/// precedence over disk content when reading through the [`FileSystem`]. /// This is the key abstraction that makes the sharing between Session -/// and WorkspaceFileSystem explicit and type-safe. +/// and [`WorkspaceFileSystem`] explicit and type-safe. +/// +/// The [`WorkspaceFileSystem`] holds a clone of this structure and checks +/// it before falling back to disk reads. #[derive(Clone, Debug)] pub struct Buffers { inner: Arc>, diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 8ef755f..3f45786 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -66,10 +66,13 @@ impl FileSystem for OsFileSystem { /// LSP file system that intercepts reads for buffered files /// -/// This implements Ruff's two-layer architecture where Layer 1 (open buffers) +/// This implements Ruff's two-layer architecture where Layer 1 (open [`Buffers`]) /// takes precedence over Layer 2 (Salsa database). When a file is read, -/// this system first checks for a buffer (in-memory content) and returns -/// that content. If no buffer exists, it falls back to reading from disk. +/// this system first checks for a buffer (in-memory content from [`TextDocument`]) +/// and returns that content. If no buffer exists, it falls back to reading from disk. +/// +/// This type is used by the [`Database`] to ensure all file reads go through +/// the buffer system first. pub struct WorkspaceFileSystem { /// In-memory buffers that take precedence over disk files buffers: Buffers, diff --git a/crates/djls-workspace/src/language.rs b/crates/djls-workspace/src/language.rs index f92811c..ea9b383 100644 --- a/crates/djls-workspace/src/language.rs +++ b/crates/djls-workspace/src/language.rs @@ -8,7 +8,8 @@ use crate::FileKind; /// Language identifier as reported by the LSP client. /// /// These identifiers follow VS Code's language ID conventions and determine -/// which analyzers and features are available for a document. +/// which analyzers and features are available for a document. Converts to +/// [`FileKind`] to route files to appropriate analyzers (Python vs Template). #[derive(Clone, Debug, PartialEq)] pub enum LanguageId { Html, diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 515df66..9d64f69 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -10,6 +10,20 @@ use url::Url; /// Convert a file:// URL to a `PathBuf` /// /// Handles percent-encoding and platform-specific path formats (e.g., Windows drives). +/// +/// # Returns +/// +/// Returns `None` if the URL scheme is not "file" or if decoding fails. +/// +/// # Examples +/// +/// ``` +/// # use url::Url; +/// # use djls_workspace::paths::url_to_path; +/// let url = Url::parse("file:///home/user/file.txt").unwrap(); +/// let path = url_to_path(&url).unwrap(); +/// assert_eq!(path.to_str().unwrap(), "/home/user/file.txt"); +/// ``` #[must_use] pub fn url_to_path(url: &Url) -> Option { // Only handle file:// URLs @@ -34,9 +48,14 @@ pub fn url_to_path(url: &Url) -> Option { /// Convert an LSP URI to a `PathBuf` /// /// This is a convenience wrapper that parses the LSP URI string and converts it. -pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { +/// +/// # Returns +/// +/// Returns `None` if the URI cannot be parsed as a URL or is not a file:// URI. +#[must_use] +pub fn lsp_uri_to_path(uri: &lsp_types::Uri) -> Option { // Parse the URI string as a URL - let url = Url::parse(lsp_uri.as_str()).ok()?; + let url = Url::parse(uri.as_str()).ok()?; url_to_path(&url) } From f6e7f9084e49da9520a587dbed388242f8fcbe4c Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 16:02:39 -0500 Subject: [PATCH 18/30] Fix missing backticks in documentation Fixed unclosed HTML tag warnings by adding backticks around: - Generic types like Arc - Type names in documentation like StorageHandle - The word 'Arc' when referring to the type --- crates/djls-workspace/src/db.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 1892e2d..03477a5 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -27,14 +27,14 @@ //! //! ## StorageHandle Pattern (for tower-lsp) //! - Database itself is NOT Send+Sync (due to RefCell in Salsa's Storage) -//! - StorageHandle IS Send+Sync, enabling use across threads +//! - `StorageHandle` IS Send+Sync, enabling use across threads //! - Session stores StorageHandle, creates Database instances on-demand //! //! ## Why Files are in Database, Overlays in Session //! - Files need persistent tracking across all queries (thus in Database) //! - Overlays are LSP-specific and change frequently (thus in Session) //! - This separation prevents Salsa invalidation cascades on every keystroke -//! - Both are accessed via Arc for thread safety and cheap cloning +//! - Both are accessed via `Arc` for thread safety and cheap cloning //! //! # Data Flow //! @@ -49,7 +49,7 @@ //! This design achieves: //! - Fast overlay updates (no Salsa invalidation) //! - Proper incremental computation (via revision tracking) -//! - Thread safety (via Arc and StorageHandle) +//! - Thread safety (via `Arc` and StorageHandle) //! - Clean separation of concerns (LSP vs computation) use std::path::{Path, PathBuf}; @@ -168,7 +168,7 @@ impl Database { /// Get or create a SourceFile for the given path. /// /// This method implements Ruff's pattern for lazy file creation. Files are created - /// with an initial revision of 0 and tracked in the Database's DashMap. The Arc + /// with an initial revision of 0 and tracked in the Database's `DashMap`. The `Arc` /// ensures cheap cloning while maintaining thread safety. pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(&path) { From af8820b7bcb94660a9b494924e99e15b76abfaca Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 16:57:14 -0500 Subject: [PATCH 19/30] Fix outdated and incorrect db.rs documentation The documentation was completely out of sync with the code: - Referenced 'overlays in Session using Arc' when they're now in Buffers - Mentioned 'LspFileSystem' which was renamed to WorkspaceFileSystem - Was overly verbose without being helpful - 'vomit of words' Rewrote to be concise and accurate: - Correctly describes the current two-layer architecture - Focuses on the critical revision dependency trick - Removes outdated implementation details - Uses proper cross-references --- crates/djls-workspace/src/buffers.rs | 20 ++++----- crates/djls-workspace/src/db.rs | 64 ++++++++-------------------- crates/djls-workspace/src/fs.rs | 3 -- 3 files changed, 26 insertions(+), 61 deletions(-) diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index ae42916..0365a3a 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -1,8 +1,8 @@ //! Shared buffer storage for open documents //! -//! This module provides the `Buffers` type which represents the in-memory -//! content of open files. These buffers are shared between the Session -//! (which manages document lifecycle) and the WorkspaceFileSystem (which +//! This module provides the [`Buffers`] type which represents the in-memory +//! content of open files. These buffers are shared between the `Session` +//! (which manages document lifecycle) and the [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) (which //! reads from them). use dashmap::DashMap; @@ -11,22 +11,24 @@ use url::Url; use crate::document::TextDocument; -/// Shared buffer storage between Session and FileSystem +/// Shared buffer storage between `Session` and [`FileSystem`]. /// /// Buffers represent the in-memory content of open files that takes /// precedence over disk content when reading through the [`FileSystem`]. /// This is the key abstraction that makes the sharing between Session /// and [`WorkspaceFileSystem`] explicit and type-safe. -/// +/// /// The [`WorkspaceFileSystem`] holds a clone of this structure and checks /// it before falling back to disk reads. +/// +/// [`FileSystem`]: crate::fs::FileSystem +/// [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem #[derive(Clone, Debug)] pub struct Buffers { inner: Arc>, } impl Buffers { - /// Create a new empty buffer storage #[must_use] pub fn new() -> Self { Self { @@ -34,23 +36,19 @@ impl Buffers { } } - /// Open a document in the buffers pub fn open(&self, url: Url, document: TextDocument) { self.inner.insert(url, document); } - /// Update an open document pub fn update(&self, url: Url, document: TextDocument) { self.inner.insert(url, document); } - /// Close a document and return it if it was open #[must_use] pub fn close(&self, url: &Url) -> Option { self.inner.remove(url).map(|(_, doc)| doc) } - /// Get a document if it's open #[must_use] pub fn get(&self, url: &Url) -> Option { self.inner.get(url).map(|entry| entry.clone()) @@ -62,7 +60,6 @@ impl Buffers { self.inner.contains_key(url) } - /// Iterate over all open buffers (for debugging) pub fn iter(&self) -> impl Iterator + '_ { self.inner .iter() @@ -75,4 +72,3 @@ impl Default for Buffers { Self::new() } } - diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 03477a5..4af7fef 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -1,56 +1,28 @@ -//! Salsa database and input entities for workspace. +//! Salsa database for incremental computation. //! -//! This module implements a two-layer architecture inspired by Ruff's design pattern -//! for efficient LSP document management with Salsa incremental computation. +//! This module provides the [`Database`] which integrates with Salsa for +//! incremental computation of Django template parsing and analysis. //! -//! # Two-Layer Architecture +//! ## Architecture //! -//! ## Layer 1: LSP Document Management (in Session) -//! - Stores overlays in `Session` using `Arc>` -//! - TextDocument contains actual content, version, language_id -//! - Changes are immediate, no Salsa invalidation on every keystroke -//! - Thread-safe via DashMap for tower-lsp's Send+Sync requirements +//! The system uses a two-layer approach: +//! 1. **Buffer layer** ([`crate::Buffers`]) - Stores open document content in memory +//! 2. **Salsa layer** ([`Database`]) - Tracks files and computes derived queries //! -//! ## Layer 2: Salsa Incremental Computation (in Database) -//! - Database is pure Salsa, no file content storage -//! - Files tracked via `Arc>` for O(1) lookups -//! - SourceFile inputs only have path and revision (no text) -//! - Content read lazily through FileSystem trait -//! - LspFileSystem intercepts reads, returns overlay or disk content +//! When Salsa needs file content, it calls [`source_text`] which: +//! 1. Creates a dependency on the file's revision (critical!) +//! 2. Reads through [`crate::WorkspaceFileSystem`] which checks buffers first +//! 3. Falls back to disk if no buffer exists //! -//! # Critical Implementation Details +//! ## The Revision Dependency //! -//! ## The Revision Dependency Trick -//! The `source_text` tracked function MUST call `file.revision(db)` to create -//! the Salsa dependency chain. Without this, revision changes won't trigger -//! invalidation of dependent queries. +//! The [`source_text`] function **must** call `file.revision(db)` to create +//! a Salsa dependency. Without this, revision changes won't invalidate queries: //! -//! ## StorageHandle Pattern (for tower-lsp) -//! - Database itself is NOT Send+Sync (due to RefCell in Salsa's Storage) -//! - `StorageHandle` IS Send+Sync, enabling use across threads -//! - Session stores StorageHandle, creates Database instances on-demand -//! -//! ## Why Files are in Database, Overlays in Session -//! - Files need persistent tracking across all queries (thus in Database) -//! - Overlays are LSP-specific and change frequently (thus in Session) -//! - This separation prevents Salsa invalidation cascades on every keystroke -//! - Both are accessed via `Arc` for thread safety and cheap cloning -//! -//! # Data Flow -//! -//! 1. **did_open/did_change** → Update overlays in Session -//! 2. **notify_file_changed()** → Bump revision, tell Salsa something changed -//! 3. **Salsa query executes** → Calls source_text() -//! 4. **source_text() calls file.revision(db)** → Creates dependency -//! 5. **source_text() calls db.read_file_content()** → Goes through FileSystem -//! 6. **LspFileSystem intercepts** → Returns overlay if exists, else disk -//! 7. **Query gets content** → Without knowing about LSP/overlays -//! -//! This design achieves: -//! - Fast overlay updates (no Salsa invalidation) -//! - Proper incremental computation (via revision tracking) -//! - Thread safety (via `Arc` and StorageHandle) -//! - Clean separation of concerns (LSP vs computation) +//! ```ignore +//! let _ = file.revision(db); // Creates the dependency chain! +//! ``` + use std::path::{Path, PathBuf}; use std::sync::Arc; diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 3f45786..c3c9ef9 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -10,9 +10,6 @@ use std::sync::Arc; use crate::{buffers::Buffers, paths}; /// Trait for file system operations -/// -/// This follows Ruff's pattern of abstracting file system operations behind a trait, -/// allowing different implementations for testing, in-memory operation, and real file access. pub trait FileSystem: Send + Sync { /// Read the entire contents of a file fn read_to_string(&self, path: &Path) -> io::Result; From c685f53dec6e922d8521d04e91bd2cc471fdbb96 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 17:05:38 -0500 Subject: [PATCH 20/30] Clean up more outdated documentation in db.rs - Removed references to 'LspFileSystem' (now WorkspaceFileSystem) - Removed verbose 'overlay' explanations (now using Buffers abstraction) - Simplified struct field documentation to be accurate and concise - Removed unnecessary mentions of 'Ruff's pattern' everywhere The documentation now accurately reflects the current implementation without verbose explanations of outdated architecture. --- crates/djls-workspace/src/db.rs | 47 +++++++++++++-------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 4af7fef..85d571d 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -6,12 +6,12 @@ //! ## Architecture //! //! The system uses a two-layer approach: -//! 1. **Buffer layer** ([`crate::Buffers`]) - Stores open document content in memory +//! 1. **Buffer layer** ([`Buffers`]) - Stores open document content in memory //! 2. **Salsa layer** ([`Database`]) - Tracks files and computes derived queries //! //! When Salsa needs file content, it calls [`source_text`] which: //! 1. Creates a dependency on the file's revision (critical!) -//! 2. Reads through [`crate::WorkspaceFileSystem`] which checks buffers first +//! 2. Reads through [`WorkspaceFileSystem`] which checks buffers first //! 3. Falls back to disk if no buffer exists //! //! ## The Revision Dependency @@ -22,7 +22,9 @@ //! ```ignore //! let _ = file.revision(db); // Creates the dependency chain! //! ``` - +//! +//! [`Buffers`]: crate::buffers::Buffers +//! [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -36,36 +38,29 @@ use crate::{FileKind, FileSystem}; /// Database trait that provides file system access for Salsa queries #[salsa::db] pub trait Db: salsa::Database { - /// Get the file system for reading files (with overlay support) + /// Get the file system for reading files. fn fs(&self) -> Option>; - /// Read file content through the file system - /// This is the primary way Salsa queries should read files, as it - /// automatically checks overlays before falling back to disk. + /// Read file content through the file system. + /// + /// Checks buffers first via [`crate::WorkspaceFileSystem`], then falls back to disk. fn read_file_content(&self, path: &Path) -> std::io::Result; } -/// Salsa database root for workspace +/// Salsa database for incremental computation. /// -/// The [`Database`] provides default storage and, in tests, captures Salsa events for -/// reuse/diagnostics. It serves as the core incremental computation engine, tracking -/// dependencies and invalidations across all inputs and derived queries. -/// -/// The database integrates with the FileSystem abstraction to read files through -/// the LspFileSystem, which automatically checks overlays before falling back to disk. +/// Tracks files and computes derived queries incrementally. Integrates with +/// [`crate::WorkspaceFileSystem`] to read file content, which checks buffers +/// before falling back to disk. #[salsa::db] #[derive(Clone)] pub struct Database { storage: salsa::Storage, - /// FileSystem integration for reading files (with overlay support) - /// This allows the database to read files through LspFileSystem, which - /// automatically checks for overlays before falling back to disk files. + /// File system for reading file content (checks buffers first, then disk). fs: Option>, - /// File tracking outside of Salsa but within Database (Arc for cheap cloning). - /// This follows Ruff's pattern where files are tracked in the Database struct - /// but not as part of Salsa's storage, enabling cheap clones via Arc. + /// Maps paths to [`SourceFile`] entities for O(1) lookup. files: Arc>, // The logs are only used for testing and demonstrating reuse: @@ -126,9 +121,7 @@ impl Database { } } - /// Read file content through the file system - /// This is the primary way Salsa queries should read files, as it - /// automatically checks overlays before falling back to disk. + /// Read file content through the file system. pub fn read_file_content(&self, path: &Path) -> std::io::Result { if let Some(fs) = &self.fs { fs.read_to_string(path) @@ -214,12 +207,10 @@ pub struct SourceFile { pub revision: u64, } -/// Read file content through the FileSystem, creating proper Salsa dependencies. +/// Read file content, creating a Salsa dependency on the file's revision. /// -/// This is the CRITICAL function that implements Ruff's two-layer architecture. -/// The call to `file.revision(db)` creates a Salsa dependency, ensuring that -/// when the revision changes, this function (and all dependent queries) are -/// invalidated and re-executed. +/// **Critical**: The call to `file.revision(db)` creates the dependency chain. +/// Without it, revision changes won't trigger query invalidation. #[salsa::tracked] pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { // This line creates the Salsa dependency on revision! Without this call, From 361d7e25983e99986624524bd60f7ea3d1945618 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 21:06:52 -0500 Subject: [PATCH 21/30] stuf and thinggs --- crates/djls-server/src/server.rs | 67 ++++----- crates/djls-server/src/session.rs | 156 +++++--------------- crates/djls-server/tests/lsp_integration.rs | 21 +-- crates/djls-workspace/src/db.rs | 64 ++------ crates/djls-workspace/src/document.rs | 2 +- crates/djls-workspace/src/fs.rs | 140 +++++++++--------- crates/djls-workspace/src/lib.rs | 6 +- crates/djls-workspace/src/paths.rs | 28 +--- 8 files changed, 153 insertions(+), 331 deletions(-) diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index daa64ac..efb3bc8 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -1,6 +1,7 @@ use std::future::Future; use std::sync::Arc; +use djls_workspace::paths; use tokio::sync::RwLock; use tower_lsp_server::jsonrpc::Result as LspResult; use tower_lsp_server::lsp_types; @@ -229,21 +230,19 @@ impl LanguageServer for DjangoLanguageServer { let new_version = params.text_document.version; let changes = params.content_changes; - if let Some(mut document) = session.get_overlay(&url) { - document.update(changes, new_version); - session.update_document(url, document); - } else { - // No existing overlay - shouldn't normally happen - tracing::warn!("Received change for document without overlay: {}", url); - - // Handle full content changes only for recovery - if let Some(change) = changes.into_iter().next() { - let document = djls_workspace::TextDocument::new( - change.text, - new_version, - djls_workspace::LanguageId::Other, - ); - session.update_document(url, document); + match session.apply_document_changes(&url, changes.clone(), new_version) { + Ok(()) => {} + Err(err) => { + tracing::warn!("{}", err); + // Recovery: handle full content changes only + if let Some(change) = changes.into_iter().next() { + let document = djls_workspace::TextDocument::new( + change.text, + new_version, + djls_workspace::LanguageId::Other, + ); + session.update_document(&url, document); + } } } }) @@ -269,40 +268,24 @@ impl LanguageServer for DjangoLanguageServer { params: lsp_types::CompletionParams, ) -> LspResult> { let response = self - .with_session(|session| { + .with_session_mut(|session| { let lsp_uri = ¶ms.text_document_position.text_document.uri; let url = Url::parse(&lsp_uri.to_string()).expect("Valid URI from LSP"); let position = params.text_document_position.position; tracing::debug!("Completion requested for {} at {:?}", url, position); - // Check if we have an overlay for this document - if let Some(document) = session.get_overlay(&url) { - tracing::debug!("Using overlay content for completion in {}", url); - - // Use the overlay content for completion - // For now, we'll return None, but this is where completion logic would go - // The key point is that we're using overlay content, not disk content - let _content = document.content(); - let _version = document.version(); - - // TODO: Implement actual completion logic using overlay content - // This would involve: - // 1. Getting context around the cursor position - // 2. Analyzing the Django template or Python content - // 3. Returning appropriate completions - - None - } else { - tracing::debug!("No overlay found for {}, using disk content", url); - - // No overlay - would use disk content via the file system - // The LspFileSystem will automatically fall back to disk - // when no overlay is available - - // TODO: Implement completion using file system content - None + if let Some(path) = paths::url_to_path(&url) { + let content = session.file_content(path); + if content.is_empty() { + tracing::debug!("File {} has no content", url); + } else { + tracing::debug!("Using content for completion in {}", url); + // TODO: Implement actual completion logic using content + } } + + None }) .await; diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index d0ddafa..2be2abd 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -164,15 +164,11 @@ impl Session { .workspace_folders .as_ref() .and_then(|folders| folders.first()) - .and_then(|folder| Self::uri_to_pathbuf(&folder.uri)) + .and_then(|folder| paths::lsp_uri_to_path(&folder.uri)) }) } - /// Converts a `file:` URI into an absolute `PathBuf`. - fn uri_to_pathbuf(uri: &lsp_types::Uri) -> Option { - paths::lsp_uri_to_path(uri) - } - + #[must_use] pub fn project(&self) -> Option<&DjangoProject> { self.project.as_ref() } @@ -181,6 +177,7 @@ impl Session { &mut self.project } + #[must_use] pub fn settings(&self) -> &Settings { &self.settings } @@ -189,70 +186,19 @@ impl Session { self.settings = settings; } - /// Get a database instance from the session. - /// - /// This creates a usable database from the handle, which can be used - /// to query and update data. The database itself is not Send/Sync, - /// but the `StorageHandle` is, allowing us to work with tower-lsp-server. - /// - /// The database will read files through the LspFileSystem, which - /// automatically returns overlay content when available. - /// - /// CRITICAL: We pass the shared files Arc to preserve file tracking - /// across Database reconstructions from StorageHandle. - #[allow(dead_code)] - pub fn db(&self) -> Database { - let storage = self.db_handle.clone().into_storage(); - Database::from_storage(storage, self.file_system.clone(), self.files.clone()) - } - - /// Get access to the file system (for Salsa integration) - #[allow(dead_code)] - pub fn file_system(&self) -> Arc { - self.file_system.clone() - } - - /// Set or update a buffer for the given document URL - /// - /// This implements Layer 1 of Ruff's architecture - storing in-memory - /// document changes that take precedence over disk content. - #[allow(dead_code)] // Used in tests - pub fn set_overlay(&self, url: Url, document: TextDocument) { - self.buffers.open(url, document); - } - - /// Remove a buffer for the given document URL - /// - /// After removal, file reads will fall back to disk content. - #[allow(dead_code)] // Used in tests - pub fn remove_overlay(&self, url: &Url) -> Option { - self.buffers.close(url) - } - - /// Check if a buffer exists for the given URL - #[allow(dead_code)] - pub fn has_overlay(&self, url: &Url) -> bool { - self.buffers.contains(url) - } - - /// Get a copy of a buffered document - pub fn get_overlay(&self, url: &Url) -> Option { - self.buffers.get(url) - } - /// Takes exclusive ownership of the database handle for mutation operations. /// /// This method extracts the `StorageHandle` from the session, replacing it /// with a temporary placeholder. This ensures there's exactly one handle /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. /// - /// # Why Not Clone? + /// ## Why Not Clone? /// /// Cloning would create multiple handles. When Salsa needs to mutate inputs, /// it calls `cancel_others()` which waits for all handles to drop. With /// multiple handles, this wait would never complete → deadlock. /// - /// # Panics + /// ## Panics /// /// This is an internal method that should only be called by `with_db_mut`. /// Multiple concurrent calls would panic when trying to take an already-taken handle. @@ -272,12 +218,13 @@ impl Session { /// Execute a closure with mutable access to the database. /// /// This method implements Salsa's required protocol for mutations: - /// 1. Takes exclusive ownership of the StorageHandle (no clones exist) + /// 1. Takes exclusive ownership of the [`StorageHandle`](salsa::StorageHandle) + /// (no clones exist) /// 2. Creates a temporary Database for the operation /// 3. Executes your closure with `&mut Database` /// 4. Extracts and restores the updated handle /// - /// # Example + /// ## Example /// /// ```rust,ignore /// session.with_db_mut(|db| { @@ -286,7 +233,7 @@ impl Session { /// }); /// ``` /// - /// # Why This Pattern? + /// ## Why This Pattern? /// /// This ensures that when Salsa needs to modify inputs (via setters like /// `set_revision`), it has exclusive access. The internal `cancel_others()` @@ -312,11 +259,11 @@ impl Session { /// Execute a closure with read-only access to the database. /// - /// For read-only operations, we can safely clone the `StorageHandle` + /// For read-only operations, we can safely clone the [`StorageHandle`](salsa::StorageHandle) /// since Salsa allows multiple concurrent readers. This is more /// efficient than taking exclusive ownership. /// - /// # Example + /// ## Example /// /// ```rust,ignore /// let content = session.with_db(|db| { @@ -382,7 +329,7 @@ impl Session { /// This method coordinates both layers: /// - Layer 1: Updates the document content in buffers /// - Layer 2: Bumps the file revision to trigger Salsa invalidation - pub fn update_document(&mut self, url: Url, document: TextDocument) { + pub fn update_document(&mut self, url: &Url, document: TextDocument) { let version = document.version(); tracing::debug!("Updating document: {} (version {})", url, version); @@ -390,8 +337,29 @@ impl Session { self.buffers.update(url.clone(), document); // Layer 2: Bump revision to trigger invalidation - if let Some(path) = paths::url_to_path(&url) { - self.notify_file_changed(path); + if let Some(path) = paths::url_to_path(url) { + self.notify_file_changed(&path); + } + } + + /// Apply incremental changes to an open document. + /// + /// This encapsulates the full update cycle: retrieving the document, + /// applying changes, updating the buffer, and bumping Salsa revision. + /// + /// Returns an error if the document is not currently open. + pub fn apply_document_changes( + &mut self, + url: &Url, + changes: Vec, + new_version: i32, + ) -> Result<(), String> { + if let Some(mut document) = self.buffers.get(url) { + document.update(changes, new_version); + self.update_document(url, document); + Ok(()) + } else { + Err(format!("Document not open: {url}")) } } @@ -418,7 +386,7 @@ impl Session { // Layer 2: Bump revision to trigger re-read from disk // We keep the file alive for potential re-opening if let Some(path) = paths::url_to_path(url) { - self.notify_file_changed(path); + self.notify_file_changed(&path); } removed @@ -428,12 +396,12 @@ impl Session { /// /// This bumps the file's revision number in Salsa, which triggers /// invalidation of any queries that depend on the file's content. - fn notify_file_changed(&mut self, path: PathBuf) { + fn notify_file_changed(&mut self, path: &Path) { self.with_db_mut(|db| { // Only bump revision if file is already being tracked // We don't create files just for notifications - if db.has_file(&path) { - let file = db.get_or_create_file(path.clone()); + if db.has_file(path) { + let file = db.get_or_create_file(path.to_path_buf()); let current_rev = file.revision(db); let new_rev = current_rev + 1; file.set_revision(db).to(new_rev); @@ -518,50 +486,6 @@ mod tests { use super::*; use djls_workspace::LanguageId; - #[test] - fn test_session_overlay_management() { - let session = Session::default(); - - let url = Url::parse("file:///test/file.py").unwrap(); - let document = TextDocument::new("print('hello')".to_string(), 1, LanguageId::Python); - - // Initially no overlay - assert!(!session.has_overlay(&url)); - assert!(session.get_overlay(&url).is_none()); - - // Set overlay - session.set_overlay(url.clone(), document.clone()); - assert!(session.has_overlay(&url)); - - let retrieved = session.get_overlay(&url).unwrap(); - assert_eq!(retrieved.content(), document.content()); - assert_eq!(retrieved.version(), document.version()); - - // Remove overlay - let removed = session.remove_overlay(&url).unwrap(); - assert_eq!(removed.content(), document.content()); - assert!(!session.has_overlay(&url)); - } - - #[test] - fn test_session_two_layer_architecture() { - let session = Session::default(); - - // Verify we have both layers - let _filesystem = session.file_system(); // Layer 2: FileSystem bridge - let _db = session.db(); // Layer 2: Salsa database - - // Verify overlay operations work (Layer 1) - let url = Url::parse("file:///test/integration.py").unwrap(); - let document = TextDocument::new("# Layer 1 content".to_string(), 1, LanguageId::Python); - - session.set_overlay(url.clone(), document); - assert!(session.has_overlay(&url)); - - // FileSystem should now return overlay content through LspFileSystem - // (This would be tested more thoroughly in integration tests) - } - #[test] fn test_revision_invalidation_chain() { use std::path::PathBuf; @@ -590,7 +514,7 @@ mod tests { println!("**[test]** Update document with new content"); let updated_document = TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); - session.update_document(url.clone(), updated_document); + session.update_document(&url, updated_document); // Read content again (should get new overlay content due to invalidation) println!( diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index 4bfc666..fa5613f 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -138,14 +138,6 @@ impl TestServer { std::fs::write(path, content).expect("Failed to write test file"); } - /// Check if a file has an overlay in the session - async fn has_overlay(&self, file_name: &str) -> bool { - let url = self.workspace_url(file_name); - self.server - .with_session(|session| session.get_overlay(&url).is_some()) - .await - } - /// Get the revision of a file async fn get_file_revision(&self, file_name: &str) -> Option { let path = self.workspace_file(file_name); @@ -168,9 +160,6 @@ async fn test_full_lsp_lifecycle() { .open_document(file_name, "

Overlay Content

", 1) .await; - // Verify overlay exists - assert!(server.has_overlay(file_name).await); - // Verify overlay content is returned (not disk content) let content = server.get_file_content(file_name).await; assert_eq!(content, "

Overlay Content

"); @@ -195,10 +184,7 @@ async fn test_full_lsp_lifecycle() { // 3. Test did_close removes overlay and bumps revision server.close_document(file_name).await; - // Verify overlay is removed - assert!(!server.has_overlay(file_name).await); - - // Verify content now comes from disk + // Verify content now comes from disk (empty since file doesn't exist) let content = server.get_file_content(file_name).await; assert_eq!(content, "

Disk Content

"); @@ -283,11 +269,6 @@ async fn test_multiple_documents_independent() { server.open_document("file2.html", "Content 2", 1).await; server.open_document("file3.html", "Content 3", 1).await; - // Verify all have overlays - assert!(server.has_overlay("file1.html").await); - assert!(server.has_overlay("file2.html").await); - assert!(server.has_overlay("file3.html").await); - // Change one document server.change_document("file2.html", "Updated 2", 2).await; diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 85d571d..36c52ce 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -42,16 +42,17 @@ pub trait Db: salsa::Database { fn fs(&self) -> Option>; /// Read file content through the file system. - /// - /// Checks buffers first via [`crate::WorkspaceFileSystem`], then falls back to disk. + /// + /// Checks buffers first via [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem), + /// then falls back to disk. fn read_file_content(&self, path: &Path) -> std::io::Result; } /// Salsa database for incremental computation. /// /// Tracks files and computes derived queries incrementally. Integrates with -/// [`crate::WorkspaceFileSystem`] to read file content, which checks buffers -/// before falling back to disk. +/// [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) to read file content, +/// which checks buffers before falling back to disk. #[salsa::db] #[derive(Clone)] pub struct Database { @@ -130,10 +131,10 @@ impl Database { } } - /// Get or create a SourceFile for the given path. + /// Get or create a [`SourceFile`] for the given path. /// /// This method implements Ruff's pattern for lazy file creation. Files are created - /// with an initial revision of 0 and tracked in the Database's `DashMap`. The `Arc` + /// with an initial revision of 0 and tracked in the [`Database`]'s `DashMap`. The `Arc` /// ensures cheap cloning while maintaining thread safety. pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(&path) { @@ -161,7 +162,7 @@ impl Database { /// Get a reference to the storage for handle extraction. /// - /// This is used by Session to extract the StorageHandle after mutations. + /// This is used by `Session` to extract the [`StorageHandle`](salsa::StorageHandle) after mutations. pub fn storage(&self) -> &salsa::Storage { &self.storage } @@ -363,58 +364,11 @@ mod tests { use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; + use crate::fs::InMemoryFileSystem; use crate::fs::WorkspaceFileSystem; use crate::language::LanguageId; use dashmap::DashMap; use salsa::Setter; - use std::collections::HashMap; - use std::io; - - // Simple in-memory filesystem for testing - struct InMemoryFileSystem { - files: HashMap, - } - - impl InMemoryFileSystem { - fn new() -> Self { - Self { - files: HashMap::new(), - } - } - - fn add_file(&mut self, path: PathBuf, content: String) { - self.files.insert(path, content); - } - } - - impl FileSystem for InMemoryFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) - } - - fn exists(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - Ok(vec![]) - } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new(io::ErrorKind::Unsupported, "Not supported")) - } - } #[test] fn test_parse_template_with_overlay() { diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 9def945..3d3b876 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -29,7 +29,7 @@ pub struct TextDocument { } impl TextDocument { - /// Create a new TextDocument with the given content + /// Create a new [`TextDocument`] with the given content #[must_use] pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { let line_index = LineIndex::new(&content); diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index c3c9ef9..0f60040 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -1,10 +1,12 @@ //! File system abstraction following Ruff's pattern //! -//! This module provides the `FileSystem` trait that abstracts file I/O operations. +//! This module provides the [`FileSystem`] trait that abstracts file I/O operations. //! This allows the LSP to work with both real files and in-memory overlays. +#[cfg(test)] +use std::collections::HashMap; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use crate::{buffers::Buffers, paths}; @@ -24,13 +26,67 @@ pub trait FileSystem: Send + Sync { fn is_directory(&self, path: &Path) -> bool; /// List directory contents - fn read_directory(&self, path: &Path) -> io::Result>; + fn read_directory(&self, path: &Path) -> io::Result>; /// Get file metadata (size, modified time, etc.) fn metadata(&self, path: &Path) -> io::Result; } -/// Standard file system implementation that uses `std::fs` +/// In-memory file system for testing +#[cfg(test)] +pub struct InMemoryFileSystem { + files: HashMap, +} + +#[cfg(test)] +impl InMemoryFileSystem { + pub fn new() -> Self { + Self { + files: HashMap::new(), + } + } + + pub fn add_file(&mut self, path: PathBuf, content: String) { + self.files.insert(path, content); + } +} + +#[cfg(test)] +impl FileSystem for InMemoryFileSystem { + fn read_to_string(&self, path: &Path) -> io::Result { + self.files + .get(path) + .cloned() + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + fn exists(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_file(&self, path: &Path) -> bool { + self.files.contains_key(path) + } + + fn is_directory(&self, _path: &Path) -> bool { + // Simplified for testing - no directories in memory filesystem + false + } + + fn read_directory(&self, _path: &Path) -> io::Result> { + // Simplified for testing + Ok(Vec::new()) + } + + fn metadata(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Metadata not supported in memory filesystem", + )) + } +} + +/// Standard file system implementation that uses [`std::fs`]. pub struct OsFileSystem; impl FileSystem for OsFileSystem { @@ -50,7 +106,7 @@ impl FileSystem for OsFileSystem { path.is_dir() } - fn read_directory(&self, path: &Path) -> io::Result> { + fn read_directory(&self, path: &Path) -> io::Result> { std::fs::read_dir(path)? .map(|entry| entry.map(|e| e.path())) .collect() @@ -61,15 +117,16 @@ impl FileSystem for OsFileSystem { } } -/// LSP file system that intercepts reads for buffered files +/// LSP file system that intercepts reads for buffered files. /// /// This implements Ruff's two-layer architecture where Layer 1 (open [`Buffers`]) /// takes precedence over Layer 2 (Salsa database). When a file is read, -/// this system first checks for a buffer (in-memory content from [`TextDocument`]) -/// and returns that content. If no buffer exists, it falls back to reading from disk. +/// this system first checks for a buffer (in-memory content from +/// [`TextDocument`](crate::document::TextDocument)) and returns that content. +/// If no buffer exists, it falls back to reading from disk. /// -/// This type is used by the [`Database`] to ensure all file reads go through -/// the buffer system first. +/// This type is used by the [`Database`](crate::db::Database) to ensure all file reads go +/// through the buffer system first. pub struct WorkspaceFileSystem { /// In-memory buffers that take precedence over disk files buffers: Buffers, @@ -110,7 +167,7 @@ impl FileSystem for WorkspaceFileSystem { self.disk.is_directory(path) } - fn read_directory(&self, path: &Path) -> io::Result> { + fn read_directory(&self, path: &Path) -> io::Result> { // Overlays are never directories, so just delegate self.disk.read_directory(path) } @@ -130,70 +187,15 @@ mod tests { use crate::language::LanguageId; use url::Url; - /// In-memory file system for testing - pub struct InMemoryFileSystem { - files: std::collections::HashMap, - } - - impl InMemoryFileSystem { - pub fn new() -> Self { - Self { - files: std::collections::HashMap::new(), - } - } - - pub fn add_file(&mut self, path: std::path::PathBuf, content: String) { - self.files.insert(path, content); - } - } - - impl FileSystem for InMemoryFileSystem { - fn read_to_string(&self, path: &Path) -> io::Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "File not found")) - } - - fn exists(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_file(&self, path: &Path) -> bool { - self.files.contains_key(path) - } - - fn is_directory(&self, _path: &Path) -> bool { - // Simplified for testing - no directories in memory filesystem - false - } - - fn read_directory(&self, _path: &Path) -> io::Result> { - // Simplified for testing - Ok(Vec::new()) - } - - fn metadata(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "Metadata not supported in memory filesystem", - )) - } - } - #[test] fn test_lsp_filesystem_overlay_precedence() { - // Create a memory filesystem with some content let mut memory_fs = InMemoryFileSystem::new(); memory_fs.add_file( std::path::PathBuf::from("/test/file.py"), "original content".to_string(), ); - // Create buffer storage let buffers = Buffers::new(); - - // Create LspFileSystem with memory fallback let lsp_fs = WorkspaceFileSystem::new(buffers.clone(), Arc::new(memory_fs)); // Before adding buffer, should read from fallback @@ -211,17 +213,13 @@ mod tests { #[test] fn test_lsp_filesystem_fallback_when_no_overlay() { - // Create memory filesystem let mut memory_fs = InMemoryFileSystem::new(); memory_fs.add_file( std::path::PathBuf::from("/test/file.py"), "disk content".to_string(), ); - // Create empty buffer storage let buffers = Buffers::new(); - - // Create LspFileSystem let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); // Should fall back to disk when no buffer exists @@ -231,14 +229,12 @@ mod tests { #[test] fn test_lsp_filesystem_other_operations_delegate() { - // Create memory filesystem let mut memory_fs = InMemoryFileSystem::new(); memory_fs.add_file( std::path::PathBuf::from("/test/file.py"), "content".to_string(), ); - // Create LspFileSystem let buffers = Buffers::new(); let lsp_fs = WorkspaceFileSystem::new(buffers, Arc::new(memory_fs)); diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 9e17d15..861a8ca 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -20,6 +20,8 @@ mod language; pub mod paths; mod template; +use std::path::Path; + pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; @@ -63,9 +65,9 @@ pub enum FileKind { } impl FileKind { - /// Determine `FileKind` from a file path extension. + /// Determine [`FileKind`] from a file path extension. #[must_use] - pub fn from_path(path: &std::path::Path) -> Self { + pub fn from_path(path: &Path) -> Self { match path.extension().and_then(|s| s.to_str()) { Some("py") => FileKind::Python, Some("html" | "htm") => FileKind::Template, diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index 9d64f69..df1a0df 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -7,23 +7,9 @@ use std::path::{Path, PathBuf}; use tower_lsp_server::lsp_types; use url::Url; -/// Convert a file:// URL to a `PathBuf` +/// Convert a `file://` URL to a [`PathBuf`]. /// /// Handles percent-encoding and platform-specific path formats (e.g., Windows drives). -/// -/// # Returns -/// -/// Returns `None` if the URL scheme is not "file" or if decoding fails. -/// -/// # Examples -/// -/// ``` -/// # use url::Url; -/// # use djls_workspace::paths::url_to_path; -/// let url = Url::parse("file:///home/user/file.txt").unwrap(); -/// let path = url_to_path(&url).unwrap(); -/// assert_eq!(path.to_str().unwrap(), "/home/user/file.txt"); -/// ``` #[must_use] pub fn url_to_path(url: &Url) -> Option { // Only handle file:// URLs @@ -45,21 +31,17 @@ pub fn url_to_path(url: &Url) -> Option { Some(PathBuf::from(path.as_ref())) } -/// Convert an LSP URI to a `PathBuf` +/// Convert an LSP URI to a [`PathBuf`]. /// /// This is a convenience wrapper that parses the LSP URI string and converts it. -/// -/// # Returns -/// -/// Returns `None` if the URI cannot be parsed as a URL or is not a file:// URI. #[must_use] -pub fn lsp_uri_to_path(uri: &lsp_types::Uri) -> Option { +pub fn lsp_uri_to_path(lsp_uri: &lsp_types::Uri) -> Option { // Parse the URI string as a URL - let url = Url::parse(uri.as_str()).ok()?; + let url = Url::parse(lsp_uri.as_str()).ok()?; url_to_path(&url) } -/// Convert a Path to a file:// URL +/// Convert a [`Path`] to a `file://` URL /// /// Handles both absolute and relative paths. Relative paths are resolved /// to absolute paths before conversion. From 00fef522ad3aa0eb703d1504f3242d05292c2660 Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:09:56 -0500 Subject: [PATCH 22/30] weeeee --- Cargo.lock | 1 + crates/djls-server/src/server.rs | 2 +- crates/djls-server/src/session.rs | 56 ++++++------------------------- crates/djls-workspace/Cargo.toml | 1 + crates/djls-workspace/src/db.rs | 31 +++++++++++++++++ 5 files changed, 44 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2722022..79834e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -518,6 +518,7 @@ dependencies = [ "tempfile", "tokio", "tower-lsp-server", + "tracing", "url", ] diff --git a/crates/djls-server/src/server.rs b/crates/djls-server/src/server.rs index efb3bc8..ae472a9 100644 --- a/crates/djls-server/src/server.rs +++ b/crates/djls-server/src/server.rs @@ -216,7 +216,7 @@ impl LanguageServer for DjangoLanguageServer { language_id, ); - session.open_document(url, document); + session.open_document(&url, document); }) .await; } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 2be2abd..8f4d9ea 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -41,7 +41,7 @@ use djls_workspace::{ db::{Database, SourceFile}, paths, Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, }; -use salsa::{Setter, StorageHandle}; +use salsa::StorageHandle; use tower_lsp_server::lsp_types; use url::Url; @@ -286,13 +286,13 @@ impl Session { /// This method coordinates both layers: /// - Layer 1: Stores the document content in buffers /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) - pub fn open_document(&mut self, url: Url, document: TextDocument) { + pub fn open_document(&mut self, url: &Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); // Layer 1: Set buffer self.buffers.open(url.clone(), document); - // Layer 2: Create file and bump revision if it already exists + // Layer 2: Create file and touch if it already exists // This is crucial: if the file was already read from disk, we need to // invalidate Salsa's cache so it re-reads through the buffer system if let Some(path) = paths::url_to_path(&url) { @@ -302,16 +302,8 @@ impl Session { let file = db.get_or_create_file(path.clone()); if already_exists { - // File was already read - bump revision to invalidate cache - let current_rev = file.revision(db); - let new_rev = current_rev + 1; - file.set_revision(db).to(new_rev); - tracing::debug!( - "Bumped revision for {} on open: {} -> {}", - path.display(), - current_rev, - new_rev - ); + // File was already read - touch to invalidate cache + db.touch_file(&path); } else { // New file - starts at revision 0 tracing::debug!( @@ -336,9 +328,9 @@ impl Session { // Layer 1: Update buffer self.buffers.update(url.clone(), document); - // Layer 2: Bump revision to trigger invalidation + // Layer 2: Touch file to trigger invalidation if let Some(path) = paths::url_to_path(url) { - self.notify_file_changed(&path); + self.with_db_mut(|db| db.touch_file(&path)); } } @@ -383,43 +375,15 @@ impl Session { ); } - // Layer 2: Bump revision to trigger re-read from disk + // Layer 2: Touch file to trigger re-read from disk // We keep the file alive for potential re-opening if let Some(path) = paths::url_to_path(url) { - self.notify_file_changed(&path); + self.with_db_mut(|db| db.touch_file(&path)); } removed } - /// Internal: Notify that a file's content has changed. - /// - /// This bumps the file's revision number in Salsa, which triggers - /// invalidation of any queries that depend on the file's content. - fn notify_file_changed(&mut self, path: &Path) { - self.with_db_mut(|db| { - // Only bump revision if file is already being tracked - // We don't create files just for notifications - if db.has_file(path) { - let file = db.get_or_create_file(path.to_path_buf()); - let current_rev = file.revision(db); - let new_rev = current_rev + 1; - file.set_revision(db).to(new_rev); - tracing::debug!( - "Bumped revision for {}: {} -> {}", - path.display(), - current_rev, - new_rev - ); - } else { - tracing::debug!( - "File {} not tracked, skipping revision bump", - path.display() - ); - } - }); - } - // ===== Safe Query API ===== // These methods encapsulate all Salsa interactions, preventing the // "mixed database instance" bug by never exposing SourceFile or Database. @@ -503,7 +467,7 @@ mod tests { 1, LanguageId::Other, ); - session.open_document(url.clone(), document); + session.open_document(&url, document); // Try to read content - this might be where it hangs println!("**[test]** try to read content - this might be where it hangs"); diff --git a/crates/djls-workspace/Cargo.toml b/crates/djls-workspace/Cargo.toml index b1fa2e0..e2fb358 100644 --- a/crates/djls-workspace/Cargo.toml +++ b/crates/djls-workspace/Cargo.toml @@ -15,6 +15,7 @@ percent-encoding = { workspace = true } salsa = { workspace = true } tokio = { workspace = true } tower-lsp-server = { workspace = true } +tracing = { workspace = true } url = { workspace = true } [dev-dependencies] diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 36c52ce..3bcef5f 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -32,6 +32,7 @@ use std::sync::Arc; use std::sync::Mutex; use dashmap::DashMap; +use salsa::Setter; use crate::{FileKind, FileSystem}; @@ -160,6 +161,36 @@ impl Database { self.files.contains_key(path) } + /// Touch a file to mark it as modified, triggering re-evaluation of dependent queries. + /// + /// Similar to Unix `touch`, this updates the file's revision number to signal + /// that cached query results depending on this file should be invalidated. + /// + /// This is typically called when: + /// - A file is opened in the editor (if it was previously cached from disk) + /// - A file's content is modified + /// - A file's buffer is closed (reverting to disk content) + pub fn touch_file(&mut self, path: &Path) { + // Get the file if it exists + let Some(file_ref) = self.files.get(path) else { + tracing::debug!("File {} not tracked, skipping touch", path.display()); + return; + }; + let file = *file_ref; + drop(file_ref); // Explicitly drop to release the lock + + let current_rev = file.revision(self); + let new_rev = current_rev + 1; + file.set_revision(self).to(new_rev); + + tracing::debug!( + "Touched {}: revision {} -> {}", + path.display(), + current_rev, + new_rev + ); + } + /// Get a reference to the storage for handle extraction. /// /// This is used by `Session` to extract the [`StorageHandle`](salsa::StorageHandle) after mutations. From f474f55b7a6558994f0519731fc7c0b8fd25f43a Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:11:06 -0500 Subject: [PATCH 23/30] remove --- task_order.md | 61 --------------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100644 task_order.md diff --git a/task_order.md b/task_order.md deleted file mode 100644 index 137a402..0000000 --- a/task_order.md +++ /dev/null @@ -1,61 +0,0 @@ -# Revised Task Order for Ruff Pattern Implementation - -## The Correct Architecture Understanding - -Based on Ruff expert clarification: -- **SourceFile should NOT store text content** (our current implementation is wrong) -- **File content is read on-demand** through a `source_text` tracked function -- **Overlays are never Salsa inputs**, they're read through FileSystem -- **File revision triggers invalidation**, not content changes - -## Implementation Order - -### Phase 1: Database Foundation -1. **task-129** - Complete Database FileSystem integration - - Database needs access to LspFileSystem to read files - - This enables tracked functions to read through FileSystem - -### Phase 2: Salsa Input Restructuring -2. **task-126** - Bridge Salsa queries to LspFileSystem - - Remove `text` field from SourceFile - - Add `path` and `revision` fields - - Create `source_text` tracked function - -### Phase 3: Query Updates -3. **task-95** - Update template parsing to use source_text query - - Update all queries to use `source_text(db, file)` - - Remove direct text access from SourceFile - -### Phase 4: LSP Integration -4. **task-112** - Add file revision tracking - - Bump file revision when overlays change - - This triggers Salsa invalidation - -### Phase 5: Testing -5. **task-127** - Test overlay behavior and Salsa integration - - Verify overlays work correctly - - Test invalidation behavior - -## Key Changes from Current Implementation - -Current (WRONG): -```rust -#[salsa::input] -pub struct SourceFile { - pub text: Arc, // ❌ Storing content in Salsa -} -``` - -Target (RIGHT): -```rust -#[salsa::input] -pub struct SourceFile { - pub path: PathBuf, - pub revision: u32, // ✅ Only track changes -} - -#[salsa::tracked] -pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { - // Read through FileSystem (checks overlays first) -} -``` From f7a1816de47fba092098ff22598389ac2c7f145d Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:16:31 -0500 Subject: [PATCH 24/30] remove --- ARCHITECTURE_INSIGHTS.md | 96 ---------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 ARCHITECTURE_INSIGHTS.md diff --git a/ARCHITECTURE_INSIGHTS.md b/ARCHITECTURE_INSIGHTS.md deleted file mode 100644 index b3dcdf0..0000000 --- a/ARCHITECTURE_INSIGHTS.md +++ /dev/null @@ -1,96 +0,0 @@ -# Architecture Insights from Ruff Investigation - -## Key Discovery: Two-Layer Architecture - -### The Problem -- LSP documents change frequently (every keystroke) -- Salsa invalidation is expensive -- Tower-lsp requires Send+Sync, but Salsa Database contains RefCell/UnsafeCell - -### The Solution (Ruff Pattern) - -#### Layer 1: LSP Document Management (Outside Salsa) -- Store overlays in `Session` using `Arc>` -- TextDocument contains actual content, version, language_id -- Changes are immediate, no Salsa invalidation - -#### Layer 2: Salsa Incremental Computation -- Database is pure Salsa, no file storage -- Queries read through FileSystem trait -- LspFileSystem intercepts reads, returns overlay or disk content - -### Critical Insights - -1. **Overlays NEVER become Salsa inputs directly** - - They're intercepted at FileSystem::read_to_string() time - - Salsa only knows "something changed", reads content lazily - -2. **StorageHandle Pattern (for tower-lsp)** - - Session stores `StorageHandle` not Database directly - - StorageHandle IS Send+Sync even though Database isn't - - Create Database instances on-demand: `session.db()` - -3. **File Management Location** - - WRONG: Store files in Database (what we initially did) - - RIGHT: Store overlays in Session, Database is pure Salsa - -4. **The Bridge** - - LspFileSystem has Arc to same overlays as Session - - When Salsa queries need content, they call FileSystem - - FileSystem checks overlays first, falls back to disk - -### Implementation Flow - -1. **did_open/did_change/did_close** → Update overlays in Session -2. **notify_file_changed()** → Tell Salsa something changed -3. **Salsa query executes** → Calls FileSystem::read_to_string() -4. **LspFileSystem intercepts** → Returns overlay if exists, else disk -5. **Query gets content** → Without knowing about LSP/overlays - -### Why This Works - -- Fast: Overlay updates don't trigger Salsa invalidation cascade -- Thread-safe: DashMap for overlays, StorageHandle for Database -- Clean separation: LSP concerns vs computation concerns -- Efficient: Salsa caching still works, just reads through FileSystem - -### Tower-lsp vs lsp-server - -- **Ruff uses lsp-server**: No Send+Sync requirement, can store Database directly -- **We use tower-lsp**: Requires Send+Sync, must use StorageHandle pattern -- Both achieve same result, different mechanisms - -## Critical Implementation Details (From Ruff Expert) - -### The Revision Dependency Trick - -**THE MOST CRITICAL INSIGHT**: In the `source_text` tracked function, calling `file.revision(db)` is what creates the Salsa dependency chain: - -```rust -#[salsa::tracked] -pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { - // THIS LINE IS CRITICAL - Creates Salsa dependency on revision! - let _ = file.revision(db); - - // Now read from FileSystem (checks overlays first) - db.read_file_content(file.path(db)) -} -``` - -Without that `file.revision(db)` call, revision changes won't trigger invalidation! - -### Key Implementation Points - -1. **Files have no text**: SourceFile inputs only have `path` and `revision`, never `text` -2. **Revision bumping triggers invalidation**: Change revision → source_text invalidated → dependent queries invalidated -3. **Files created lazily**: Don't pre-create, let them be created on first access -4. **Simple counters work**: Revision can be a simple u64 counter, doesn't need timestamps -5. **StorageHandle update required**: After DB modifications in LSP handlers, must update the handle - -### Common Pitfalls - -- **Forgetting the revision dependency** - Without `file.revision(db)`, nothing invalidates -- **Storing text in Salsa inputs** - Breaks the entire pattern -- **Not bumping revision on overlay changes** - Queries won't see new content -- **Creating files eagerly** - Unnecessary and inefficient - From 8a63ebc3d20af240ec78193aa03c8d63c601399c Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:16:51 -0500 Subject: [PATCH 25/30] remove --- REVISION_TRACKING_ARCHITECTURE.md | 341 ------------------------------ 1 file changed, 341 deletions(-) delete mode 100644 REVISION_TRACKING_ARCHITECTURE.md diff --git a/REVISION_TRACKING_ARCHITECTURE.md b/REVISION_TRACKING_ARCHITECTURE.md deleted file mode 100644 index 202c118..0000000 --- a/REVISION_TRACKING_ARCHITECTURE.md +++ /dev/null @@ -1,341 +0,0 @@ -# Revision Tracking Architecture for Django Language Server - -## Overview - -This document captures the complete understanding of how to implement revision tracking for task-112, based on extensive discussions with a Ruff architecture expert. The goal is to connect the Session's overlay system with Salsa's query invalidation mechanism through per-file revision tracking. - -## The Critical Breakthrough: Dual-Layer Architecture - -### The Confusion We Had - -We conflated two different concepts: -1. **Database struct** - The Rust struct that implements the Salsa database trait -2. **Salsa database** - The actual Salsa storage system with inputs/queries - -### The Key Insight - -**Database struct ≠ Salsa database** - -The Database struct can contain: -- Salsa storage (the actual Salsa database) -- Additional non-Salsa data structures (like file tracking) - -## The Architecture Pattern (From Ruff) - -### Ruff's Implementation - -```rust -// Ruff's Database contains BOTH Salsa and non-Salsa data -pub struct ProjectDatabase { - storage: salsa::Storage, // Salsa's data - files: Files, // NOT Salsa data, but in Database struct! -} - -// Files is Arc-wrapped for cheap cloning -#[derive(Clone)] -pub struct Files { - inner: Arc, // Shared across clones -} - -struct FilesInner { - system_by_path: FxDashMap, // Thread-safe -} -``` - -### Our Implementation - -```rust -// Django LS Database structure -#[derive(Clone)] -pub struct Database { - storage: salsa::Storage, - files: Arc>, // Arc makes cloning cheap! -} - -// Session still uses StorageHandle for tower-lsp -pub struct Session { - db_handle: StorageHandle, // Still needed! - overlays: Arc>, // LSP document state -} -``` - -## Why This Works with Send+Sync Requirements - -1. **Arc is Send+Sync** - Thread-safe by design -2. **Cloning is cheap** - Only clones the Arc pointer (8 bytes) -3. **Persistence across clones** - All clones share the same DashMap -4. **StorageHandle compatible** - Database remains clonable and Send+Sync - -## Implementation Details - -### 1. Database Implementation - -```rust -impl Database { - pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { - self.files - .entry(path.clone()) - .or_insert_with(|| { - // Create Salsa input with initial revision 0 - SourceFile::new(self, path, 0) - }) - .clone() - } -} - -impl Clone for Database { - fn clone(&self) -> self { - Self { - storage: self.storage.clone(), // Salsa handles this - files: self.files.clone(), // Just clones Arc! - } - } -} -``` - -### 2. The Critical Pattern for Every Overlay Change - -```rust -pub fn handle_overlay_change(session: &mut Session, url: Url, content: String) { - // 1. Extract database from StorageHandle - let mut db = session.db_handle.get(); - - // 2. Update overlay in Session - session.overlays.insert(url.clone(), TextDocument::new(content)); - - // 3. Get or create file in Database - let path = path_from_url(&url); - let file = db.get_or_create_file(path); - - // 4. Bump revision (simple incrementing counter) - let current_rev = file.revision(&db); - file.set_revision(&mut db).to(current_rev + 1); - - // 5. Update StorageHandle with modified database - session.db_handle.update(db); // CRITICAL! -} -``` - -### 3. LSP Handler Updates - -#### did_open - -```rust -pub fn did_open(&mut self, params: DidOpenTextDocumentParams) { - let mut db = self.session.db_handle.get(); - - // Set overlay - self.session.overlays.insert( - params.text_document.uri.clone(), - TextDocument::new(params.text_document.text) - ); - - // Create file with initial revision 0 - let path = path_from_url(¶ms.text_document.uri); - db.get_or_create_file(path); // Creates with revision 0 - - self.session.db_handle.update(db); -} -``` - -#### did_change - -```rust -pub fn did_change(&mut self, params: DidChangeTextDocumentParams) { - let mut db = self.session.db_handle.get(); - - // Update overlay - let new_content = params.content_changes[0].text.clone(); - self.session.overlays.insert( - params.text_document.uri.clone(), - TextDocument::new(new_content) - ); - - // Bump revision - let path = path_from_url(¶ms.text_document.uri); - let file = db.get_or_create_file(path); - let new_rev = file.revision(&db) + 1; - file.set_revision(&mut db).to(new_rev); - - self.session.db_handle.update(db); -} -``` - -#### did_close - -```rust -pub fn did_close(&mut self, params: DidCloseTextDocumentParams) { - let mut db = self.session.db_handle.get(); - - // Remove overlay - self.session.overlays.remove(¶ms.text_document.uri); - - // Bump revision to trigger re-read from disk - let path = path_from_url(¶ms.text_document.uri); - if let Some(file) = db.files.get(&path) { - let new_rev = file.revision(&db) + 1; - file.set_revision(&mut db).to(new_rev); - } - - self.session.db_handle.update(db); -} -``` - -## Key Implementation Guidelines from Ruff Expert - -### 1. File Tracking Location - -- Store in Database struct (not Session) -- Use Arc for thread-safety and cheap cloning -- This keeps file tracking close to where it's used - -### 2. Revision Management - -- Use simple incrementing counter per file (not timestamps) -- Each file has independent revision tracking -- Revision just needs to change, doesn't need to be monotonic -- Example: `file.set_revision(&mut db).to(current + 1)` - -### 3. Lazy File Creation - -Files should be created: -- On did_open (via get_or_create_file) -- On first query access if needed -- NOT eagerly for all possible files - -### 4. File Lifecycle - -- **On open**: Create file with revision 0 -- **On change**: Bump revision to trigger invalidation -- **On close**: Keep file alive, bump revision for re-read from disk -- **Never remove**: Files stay in tracking even after close - -### 5. Batch Changes for Performance - -When possible, batch multiple changes: - -```rust -pub fn apply_batch_changes(&mut self, changes: Vec) { - let mut db = self.session.db_handle.get(); - - for change in changes { - // Process each change - let file = db.get_or_create_file(change.path); - file.set_revision(&mut db).to(file.revision(&db) + 1); - } - - // Single StorageHandle update at the end - self.session.db_handle.update(db); -} -``` - -### 6. Thread Safety with DashMap - -Use DashMap's atomic entry API: - -```rust -self.files.entry(path.clone()) - .and_modify(|file| { - // Modify existing - file.set_revision(db).to(new_rev); - }) - .or_insert_with(|| { - // Create new - SourceFile::builder(path) - .revision(0) - .new(db) - }); -``` - -## Critical Pitfalls to Avoid - -1. **NOT BUMPING REVISION** - Every overlay change MUST bump revision or Salsa won't invalidate -2. **FORGETTING STORAGEHANDLE UPDATE** - Must call `session.db_handle.update(db)` after changes -3. **CREATING FILES EAGERLY** - Let files be created lazily on first access -4. **USING TIMESTAMPS** - Simple incrementing counter is sufficient -5. **REMOVING FILES** - Keep files alive even after close, just bump revision - -## The Two-Layer Model - -### Layer 1: Non-Salsa (but in Database struct) -- `Arc>` - File tracking -- Thread-safe via Arc+DashMap -- Cheap to clone via Arc -- Acts as a lookup table - -### Layer 2: Salsa Inputs -- `SourceFile` entities created via `SourceFile::new(db)` -- Have revision fields for invalidation -- Tracked by Salsa's dependency system -- Invalidation cascades through dependent queries - -## Complete Architecture Summary - -| Component | Contains | Purpose | -|-----------|----------|---------| -| **Database** | `storage` + `Arc>` | Salsa queries + file tracking | -| **Session** | `StorageHandle` + `Arc>` | LSP state + overlays | -| **StorageHandle** | `Arc>>` | Bridge for tower-lsp lifetime requirements | -| **SourceFile** | Salsa input with path + revision | Triggers query invalidation | - -## The Flow - -1. **LSP request arrives** → tower-lsp handler -2. **Extract database** → `db = session.db_handle.get()` -3. **Update overlay** → `session.overlays.insert(url, content)` -4. **Get/create file** → `db.get_or_create_file(path)` -5. **Bump revision** → `file.set_revision(&mut db).to(current + 1)` -6. **Update handle** → `session.db_handle.update(db)` -7. **Salsa invalidates** → `source_text` query re-executes -8. **Queries see new content** → Through overlay-aware FileSystem - -## Why StorageHandle is Still Essential - -1. **tower-lsp requirement**: Needs 'static lifetime for async handlers -2. **Snapshot management**: Safe extraction and update of database -3. **Thread safety**: Bridges async boundaries safely -4. **Atomic updates**: Ensures consistent state transitions - -## Testing Strategy - -1. **Revision bumping**: Verify each overlay operation bumps revision -2. **Invalidation cascade**: Ensure source_text re-executes after revision bump -3. **Thread safety**: Concurrent overlay updates work correctly -4. **Clone behavior**: Database clones share the same file tracking -5. **Lazy creation**: Files only created when accessed - -## Implementation Checklist - -- [ ] Add `Arc>` to Database struct -- [ ] Implement Clone for Database (clone both storage and Arc) -- [ ] Create `get_or_create_file` method using atomic entry API -- [ ] Update did_open to create files with revision 0 -- [ ] Update did_change to bump revision after overlay update -- [ ] Update did_close to bump revision (keep file alive) -- [ ] Ensure StorageHandle updates after all database modifications -- [ ] Add tests for revision tracking and invalidation - -## Questions That Were Answered - -1. **Q: Files in Database or Session?** - A: In Database, but Arc-wrapped for cheap cloning - -2. **Q: How does this work with Send+Sync?** - A: Arc is Send+Sync, making Database clonable and thread-safe - -3. **Q: Do we still need StorageHandle?** - A: YES! It bridges tower-lsp's lifetime requirements - -4. **Q: Timestamp or counter for revisions?** - A: Simple incrementing counter per file - -5. **Q: Remove files on close?** - A: No, keep them alive and bump revision for re-read - -## The Key Insight - -Database struct is a container that holds BOTH: -- Salsa storage (for queries and inputs) -- Non-Salsa data (file tracking via Arc) - -Arc makes the non-Salsa data cheap to clone while maintaining Send+Sync compatibility. This is the pattern Ruff uses and what we should implement. \ No newline at end of file From 84f1073a1dc280fc7b1497bfeb40dca50e8d9afc Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:17:04 -0500 Subject: [PATCH 26/30] remove --- RUFF_ARCHITECTURE_INSIGHTS.md | 77 ----------------------------------- 1 file changed, 77 deletions(-) delete mode 100644 RUFF_ARCHITECTURE_INSIGHTS.md diff --git a/RUFF_ARCHITECTURE_INSIGHTS.md b/RUFF_ARCHITECTURE_INSIGHTS.md deleted file mode 100644 index f4a0bf1..0000000 --- a/RUFF_ARCHITECTURE_INSIGHTS.md +++ /dev/null @@ -1,77 +0,0 @@ -# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution - -## This document is preserved for historical context but is OUTDATED -## We found the StorageHandle solution that solves the Send+Sync issue - -# Critical Discovery: The Tower-LSP vs lsp-server Architectural Mismatch - -## The Real Problem - -Your Ruff expert friend is correct. The fundamental issue is: - -### What We Found: - -1. **Salsa commit a3ffa22 uses `RefCell` and `UnsafeCell`** - These are inherently not `Sync` -2. **Tower-LSP requires `Sync`** - Because handlers take `&self` in async contexts -3. **Ruff uses `lsp-server`** - Which doesn't require `Sync` on the server struct - -### The Mystery: - -Your expert suggests Ruff's database IS `Send + Sync`, but our testing shows that with the same Salsa commit, the database contains: -- `RefCell` (not Sync) -- `UnsafeCell>` (not Sync) - -## Possible Explanations: - -### Theory 1: Ruff Has Custom Patches -Ruff might have additional patches or workarounds not visible in the commit hash. - -### Theory 2: Different Usage Pattern -Ruff might structure their database differently to avoid the Sync requirement entirely. - -### Theory 3: lsp-server Architecture -Since Ruff uses `lsp-server` (not `tower-lsp`), they might never need the database to be Sync: -- They clone the database for background work (requires Send only) -- The main thread owns the database, background threads get clones -- No shared references across threads - -## Verification Needed: - -1. **Check if Ruff's database is actually Sync**: - - Look for unsafe impl Sync in their codebase - - Check if they wrap the database differently - -2. **Understand lsp-server's threading model**: - - How does it handle async without requiring Sync? - - What's the message passing pattern? - -## Solution Decision Matrix (Updated): - -| Solution | Effort | Performance | Risk | Compatibility | -|----------|---------|------------|------|---------------| -| **Switch to lsp-server** | High | High | Medium | Perfect Ruff parity | -| **Actor Pattern** | Medium | Medium | Low | Works with tower-lsp | -| **Arc** | Low | Poor | Low | Works but slow | -| **Unsafe Sync wrapper** | Low | High | Very High | Dangerous | -| **Database per request** | Medium | Poor | Low | Loses memoization | - -## Recommended Action Plan: - -### Immediate (Today): -1. Verify that Salsa a3ffa22 truly has RefCell/UnsafeCell -2. Check if there are any Ruff-specific patches to Salsa -3. Test the actor pattern as a better alternative to Arc - -### Short-term (This Week): -1. Implement actor pattern if Salsa can't be made Sync -2. OR investigate unsafe Sync wrapper with careful single-threaded access guarantees - -### Long-term (This Month): -1. Consider migrating to lsp-server for full Ruff compatibility -2. OR contribute Sync support to Salsa upstream - -## The Key Insight: - -**Tower-LSP's architecture is fundamentally incompatible with Salsa's current design.** - -Ruff avoided this by using `lsp-server`, which has a different threading model that doesn't require Sync on the database. From 43138a9dd3615c9785d18070c7da94653cb83c0c Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:17:16 -0500 Subject: [PATCH 27/30] remove --- check_ruff_pattern.md | 94 ------------------------------------------- 1 file changed, 94 deletions(-) delete mode 100644 check_ruff_pattern.md diff --git a/check_ruff_pattern.md b/check_ruff_pattern.md deleted file mode 100644 index 5253b69..0000000 --- a/check_ruff_pattern.md +++ /dev/null @@ -1,94 +0,0 @@ -# OUTDATED - See ARCHITECTURE_INSIGHTS.md for current solution - -## This document is preserved for historical context but is OUTDATED -## We found the StorageHandle solution that solves the Send+Sync issue - -# Key Findings from Ruff's Architecture - -Based on the exploration, here's what we discovered: - -## Current Django LS Architecture - -### What We Have: -1. `Database` struct with `#[derive(Clone)]` and Salsa storage -2. `WorkspaceDatabase` that wraps `Database` and uses `DashMap` for thread-safe file storage -3. `Session` that owns `WorkspaceDatabase` directly (not wrapped in Arc) -4. Tower-LSP server that requires `Send + Sync` for async handlers - -### The Problem: -- `Database` is not `Sync` due to `RefCell` and `UnsafeCell` in Salsa's `ZalsaLocal` -- This prevents `Session` from being `Sync`, which breaks tower-lsp async handlers - -## Ruff's Solution (From Analysis) - -### They Don't Make Database Sync! -The key insight is that Ruff **doesn't actually make the database Send + Sync**. Instead: - -1. **Clone for Background Work**: They clone the database for each background task -2. **Move Not Share**: The cloned database is *moved* into the task (requires Send, not Sync) -3. **Message Passing**: Results are sent back via channels - -### Critical Difference: -- Ruff uses a custom LSP implementation that doesn't require `Sync` on the session -- Tower-LSP *does* require `Sync` because handlers take `&self` - -## The Real Problem - -Tower-LSP's `LanguageServer` trait requires: -```rust -async fn initialize(&self, ...) -> ... -// ^^^^^ This requires self to be Sync! -``` - -But with Salsa's current implementation, the Database can never be Sync. - -## Solution Options - -### Option 1: Wrap Database in Arc (Current Workaround) -```rust -pub struct Session { - database: Arc>, - // ... -} -``` -Downsides: Lock contention, defeats purpose of Salsa's internal optimization - -### Option 2: Move Database Out of Session -```rust -pub struct Session { - // Don't store database here - file_index: Arc>, - settings: Settings, -} - -// Create database on demand for each request -impl LanguageServer for Server { - async fn some_handler(&self) { - let db = create_database_from_index(&self.session.file_index); - // Use db for this request - } -} -``` - -### Option 3: Use Actor Pattern -```rust -pub struct DatabaseActor { - database: WorkspaceDatabase, - rx: mpsc::Receiver, -} - -pub struct Session { - db_tx: mpsc::Sender, -} -``` - -### Option 4: Custom unsafe Send/Sync implementation -This is risky but possible if we ensure single-threaded access patterns. - -## The Salsa Version Mystery - -We're using the exact same Salsa commit as Ruff, with the same features. The issue is NOT the Salsa version, but how tower-lsp forces us to use it. - -Ruff likely either: -1. Doesn't use tower-lsp (has custom implementation) -2. Or structures their server differently to avoid needing Sync on the database From f47d9dfe4dab462e5de2987768af063f8b1db64b Mon Sep 17 00:00:00 2001 From: Josh Date: Fri, 29 Aug 2025 22:18:12 -0500 Subject: [PATCH 28/30] lint --- crates/djls-server/src/client.rs | 5 ++++- crates/djls-server/src/session.rs | 18 ++++++++++++------ crates/djls-server/tests/lsp_integration.rs | 18 +++++++++++------- crates/djls-workspace/src/buffers.rs | 3 ++- crates/djls-workspace/src/db.rs | 11 +++++++---- crates/djls-workspace/src/document.rs | 5 +++-- crates/djls-workspace/src/fs.rs | 9 ++++++--- crates/djls-workspace/src/lib.rs | 4 +++- crates/djls-workspace/src/paths.rs | 4 +++- 9 files changed, 51 insertions(+), 26 deletions(-) diff --git a/crates/djls-server/src/client.rs b/crates/djls-server/src/client.rs index 11a5f62..35e616f 100644 --- a/crates/djls-server/src/client.rs +++ b/crates/djls-server/src/client.rs @@ -193,7 +193,10 @@ pub mod monitoring { } } - pub fn progress + Send>(token: lsp_types::ProgressToken, title: T) -> Option { + pub fn progress + Send>( + token: lsp_types::ProgressToken, + title: T, + ) -> Option { get_client().map(|client| client.progress(token, title)) } } diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index 8f4d9ea..be9224c 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -31,16 +31,21 @@ //! //! The explicit method names make the intent clear and prevent accidental misuse. -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use dashmap::DashMap; use djls_conf::Settings; use djls_project::DjangoProject; -use djls_workspace::{ - db::{Database, SourceFile}, - paths, Buffers, FileSystem, OsFileSystem, TextDocument, WorkspaceFileSystem, -}; +use djls_workspace::db::Database; +use djls_workspace::db::SourceFile; +use djls_workspace::paths; +use djls_workspace::Buffers; +use djls_workspace::FileSystem; +use djls_workspace::OsFileSystem; +use djls_workspace::TextDocument; +use djls_workspace::WorkspaceFileSystem; use salsa::StorageHandle; use tower_lsp_server::lsp_types; use url::Url; @@ -447,9 +452,10 @@ impl Default for Session { #[cfg(test)] mod tests { - use super::*; use djls_workspace::LanguageId; + use super::*; + #[test] fn test_revision_invalidation_chain() { use std::path::PathBuf; diff --git a/crates/djls-server/tests/lsp_integration.rs b/crates/djls-server/tests/lsp_integration.rs index fa5613f..d277275 100644 --- a/crates/djls-server/tests/lsp_integration.rs +++ b/crates/djls-server/tests/lsp_integration.rs @@ -12,11 +12,16 @@ use std::sync::Arc; use djls_server::DjangoLanguageServer; use tempfile::TempDir; -use tower_lsp_server::lsp_types::{ - DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, - InitializeParams, InitializedParams, TextDocumentContentChangeEvent, TextDocumentIdentifier, - TextDocumentItem, VersionedTextDocumentIdentifier, WorkspaceFolder, -}; +use tower_lsp_server::lsp_types::DidChangeTextDocumentParams; +use tower_lsp_server::lsp_types::DidCloseTextDocumentParams; +use tower_lsp_server::lsp_types::DidOpenTextDocumentParams; +use tower_lsp_server::lsp_types::InitializeParams; +use tower_lsp_server::lsp_types::InitializedParams; +use tower_lsp_server::lsp_types::TextDocumentContentChangeEvent; +use tower_lsp_server::lsp_types::TextDocumentIdentifier; +use tower_lsp_server::lsp_types::TextDocumentItem; +use tower_lsp_server::lsp_types::VersionedTextDocumentIdentifier; +use tower_lsp_server::lsp_types::WorkspaceFolder; use tower_lsp_server::LanguageServer; use url::Url; @@ -240,7 +245,7 @@ async fn test_template_parsing_with_overlays() { .await; use djls_workspace::db::parse_template; - // Parse template through the session + // Parse template through the session let workspace_path = server.workspace_file(file_name); let ast = server .server @@ -441,4 +446,3 @@ async fn test_revision_tracking_across_lifecycle() { server.change_document(file_name, "Final", 11).await; assert_eq!(server.get_file_revision(file_name).await, Some(7)); } - diff --git a/crates/djls-workspace/src/buffers.rs b/crates/djls-workspace/src/buffers.rs index 0365a3a..6f26ad1 100644 --- a/crates/djls-workspace/src/buffers.rs +++ b/crates/djls-workspace/src/buffers.rs @@ -5,8 +5,9 @@ //! (which manages document lifecycle) and the [`WorkspaceFileSystem`](crate::fs::WorkspaceFileSystem) (which //! reads from them). -use dashmap::DashMap; use std::sync::Arc; + +use dashmap::DashMap; use url::Url; use crate::document::TextDocument; diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index 3bcef5f..deaccc7 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -26,7 +26,8 @@ //! [`Buffers`]: crate::buffers::Buffers //! [`WorkspaceFileSystem`]: crate::fs::WorkspaceFileSystem -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; #[cfg(test)] use std::sync::Mutex; @@ -34,7 +35,8 @@ use std::sync::Mutex; use dashmap::DashMap; use salsa::Setter; -use crate::{FileKind, FileSystem}; +use crate::FileKind; +use crate::FileSystem; /// Database trait that provides file system access for Salsa queries #[salsa::db] @@ -392,14 +394,15 @@ pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { #[cfg(test)] mod tests { + use dashmap::DashMap; + use salsa::Setter; + use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::fs::InMemoryFileSystem; use crate::fs::WorkspaceFileSystem; use crate::language::LanguageId; - use dashmap::DashMap; - use salsa::Setter; #[test] fn test_parse_template_with_overlay() { diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 3d3b876..60c70bb 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -5,11 +5,12 @@ //! performance when handling frequent position-based operations like hover, completion, //! and diagnostics. +use tower_lsp_server::lsp_types::Position; +use tower_lsp_server::lsp_types::Range; + use crate::language::LanguageId; use crate::template::ClosingBrace; use crate::template::TemplateTagContext; -use tower_lsp_server::lsp_types::Position; -use tower_lsp_server::lsp_types::Range; /// In-memory representation of an open document in the LSP. /// diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index 0f60040..b0e7ac3 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -6,10 +6,12 @@ #[cfg(test)] use std::collections::HashMap; use std::io; -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; -use crate::{buffers::Buffers, paths}; +use crate::buffers::Buffers; +use crate::paths; /// Trait for file system operations pub trait FileSystem: Send + Sync { @@ -181,11 +183,12 @@ impl FileSystem for WorkspaceFileSystem { #[cfg(test)] mod tests { + use url::Url; + use super::*; use crate::buffers::Buffers; use crate::document::TextDocument; use crate::language::LanguageId; - use url::Url; #[test] fn test_lsp_filesystem_overlay_precedence() { diff --git a/crates/djls-workspace/src/lib.rs b/crates/djls-workspace/src/lib.rs index 861a8ca..b8b80e5 100644 --- a/crates/djls-workspace/src/lib.rs +++ b/crates/djls-workspace/src/lib.rs @@ -25,7 +25,9 @@ use std::path::Path; pub use buffers::Buffers; pub use db::Database; pub use document::TextDocument; -pub use fs::{FileSystem, OsFileSystem, WorkspaceFileSystem}; +pub use fs::FileSystem; +pub use fs::OsFileSystem; +pub use fs::WorkspaceFileSystem; pub use language::LanguageId; /// Stable, compact identifier for files across the subsystem. diff --git a/crates/djls-workspace/src/paths.rs b/crates/djls-workspace/src/paths.rs index df1a0df..2fde628 100644 --- a/crates/djls-workspace/src/paths.rs +++ b/crates/djls-workspace/src/paths.rs @@ -3,7 +3,9 @@ //! This module provides consistent conversion between file paths and URLs, //! handling platform-specific differences and encoding issues. -use std::path::{Path, PathBuf}; +use std::path::Path; +use std::path::PathBuf; + use tower_lsp_server::lsp_types; use url::Url; From 196a6344fed35b7a61d5056f18402975561b1358 Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 2 Sep 2025 23:19:24 -0500 Subject: [PATCH 29/30] remove comments and adjust some others --- crates/djls-workspace/src/db.rs | 52 +++------------------------ crates/djls-workspace/src/document.rs | 9 ++--- crates/djls-workspace/src/fs.rs | 5 ++- crates/djls-workspace/src/template.rs | 2 ++ 4 files changed, 14 insertions(+), 54 deletions(-) diff --git a/crates/djls-workspace/src/db.rs b/crates/djls-workspace/src/db.rs index deaccc7..0290897 100644 --- a/crates/djls-workspace/src/db.rs +++ b/crates/djls-workspace/src/db.rs @@ -61,6 +61,7 @@ pub trait Db: salsa::Database { pub struct Database { storage: salsa::Storage, + // TODO: does this need to be an Option? /// File system for reading file content (checks buffers first, then disk). fs: Option>, @@ -98,7 +99,6 @@ impl Default for Database { } impl Database { - /// Create a new database with fresh storage. pub fn new(file_system: Arc, files: Arc>) -> Self { Self { storage: salsa::Storage::new(None), @@ -109,8 +109,6 @@ impl Database { } } - /// Create a database instance from an existing storage. - /// This preserves both the file system and files Arc across database operations. pub fn from_storage( storage: salsa::Storage, file_system: Arc, @@ -136,9 +134,8 @@ impl Database { /// Get or create a [`SourceFile`] for the given path. /// - /// This method implements Ruff's pattern for lazy file creation. Files are created - /// with an initial revision of 0 and tracked in the [`Database`]'s `DashMap`. The `Arc` - /// ensures cheap cloning while maintaining thread safety. + /// Files are created with an initial revision of 0 and tracked in the [`Database`]'s + /// `DashMap`. The `Arc` ensures cheap cloning while maintaining thread safety. pub fn get_or_create_file(&mut self, path: PathBuf) -> SourceFile { if let Some(file_ref) = self.files.get(&path) { // Copy the value (SourceFile is Copy) and drop the guard immediately @@ -242,9 +239,6 @@ pub struct SourceFile { } /// Read file content, creating a Salsa dependency on the file's revision. -/// -/// **Critical**: The call to `file.revision(db)` creates the dependency chain. -/// Without it, revision changes won't trigger query invalidation. #[salsa::tracked] pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { // This line creates the Salsa dependency on revision! Without this call, @@ -260,18 +254,6 @@ pub fn source_text(db: &dyn Db, file: SourceFile) -> Arc { } } -/// Global input configuring ordered template loader roots. -/// -/// [`TemplateLoaderOrder`] represents the Django `TEMPLATES[n]['DIRS']` configuration, -/// defining the search order for template resolution. This is a global input that -/// affects template name resolution across the entire project. -#[salsa::input] -pub struct TemplateLoaderOrder { - /// Ordered list of template root directories - #[returns(ref)] - pub roots: Arc<[String]>, -} - /// Represents a file path for Salsa tracking. /// /// [`FilePath`] is a Salsa input entity that tracks a file path for use in @@ -347,7 +329,8 @@ pub fn parse_template_by_path(db: &dyn Db, file_path: FilePath) -> Option { // Convert errors to strings @@ -367,31 +350,6 @@ pub fn parse_template_by_path(db: &dyn Db, file_path: FilePath) -> Option Arc<[String]> { - parse_template_by_path(db, file_path) - .map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) -} - -/// Get template parsing errors for a file. -/// -/// This Salsa tracked function extracts just the errors from the parsed template, -/// useful for diagnostics without needing the full AST. -/// -/// Returns an empty vector for non-template files. -#[salsa::tracked] -pub fn template_errors(db: &dyn Db, file: SourceFile) -> Arc<[String]> { - parse_template(db, file).map_or_else(|| Arc::from(vec![]), |ast| Arc::from(ast.errors.clone())) -} - #[cfg(test)] mod tests { use dashmap::DashMap; diff --git a/crates/djls-workspace/src/document.rs b/crates/djls-workspace/src/document.rs index 60c70bb..eb67d47 100644 --- a/crates/djls-workspace/src/document.rs +++ b/crates/djls-workspace/src/document.rs @@ -30,7 +30,6 @@ pub struct TextDocument { } impl TextDocument { - /// Create a new [`TextDocument`] with the given content #[must_use] pub fn new(content: String, version: i32, language_id: LanguageId) -> Self { let line_index = LineIndex::new(&content); @@ -42,19 +41,16 @@ impl TextDocument { } } - /// Get the document's content #[must_use] pub fn content(&self) -> &str { &self.content } - /// Get the version number #[must_use] pub fn version(&self) -> i32 { self.version } - /// Get the language identifier #[must_use] pub fn language_id(&self) -> LanguageId { self.language_id.clone() @@ -65,6 +61,7 @@ impl TextDocument { &self.line_index } + #[must_use] pub fn get_line(&self, line: u32) -> Option { let line_start = *self.line_index.line_starts.get(line as usize)?; let line_end = self @@ -77,6 +74,7 @@ impl TextDocument { Some(self.content[line_start as usize..line_end as usize].to_string()) } + #[must_use] pub fn get_text_range(&self, range: Range) -> Option { let start_offset = self.line_index.offset(range.start)? as usize; let end_offset = self.line_index.offset(range.end)? as usize; @@ -100,6 +98,7 @@ impl TextDocument { self.version = version; } + #[must_use] pub fn get_template_tag_context(&self, position: Position) -> Option { let start = self.line_index.line_starts.get(position.line as usize)?; let end = self @@ -135,10 +134,12 @@ impl TextDocument { }) } + #[must_use] pub fn position_to_offset(&self, position: Position) -> Option { self.line_index.offset(position) } + #[must_use] pub fn offset_to_position(&self, offset: u32) -> Position { self.line_index.position(offset) } diff --git a/crates/djls-workspace/src/fs.rs b/crates/djls-workspace/src/fs.rs index b0e7ac3..00b5cb1 100644 --- a/crates/djls-workspace/src/fs.rs +++ b/crates/djls-workspace/src/fs.rs @@ -1,4 +1,4 @@ -//! File system abstraction following Ruff's pattern +//! Virtual file system abstraction //! //! This module provides the [`FileSystem`] trait that abstracts file I/O operations. //! This allows the LSP to work with both real files and in-memory overlays. @@ -121,7 +121,7 @@ impl FileSystem for OsFileSystem { /// LSP file system that intercepts reads for buffered files. /// -/// This implements Ruff's two-layer architecture where Layer 1 (open [`Buffers`]) +/// This implements a two-layer architecture where Layer 1 (open [`Buffers`]) /// takes precedence over Layer 2 (Salsa database). When a file is read, /// this system first checks for a buffer (in-memory content from /// [`TextDocument`](crate::document::TextDocument)) and returns that content. @@ -137,7 +137,6 @@ pub struct WorkspaceFileSystem { } impl WorkspaceFileSystem { - /// Create a new [`WorkspaceFileSystem`] with the given buffer storage and fallback #[must_use] pub fn new(buffers: Buffers, disk: Arc) -> Self { Self { buffers, disk } diff --git a/crates/djls-workspace/src/template.rs b/crates/djls-workspace/src/template.rs index 2ebc324..b2bd44b 100644 --- a/crates/djls-workspace/src/template.rs +++ b/crates/djls-workspace/src/template.rs @@ -3,6 +3,8 @@ //! Detects cursor position context within Django template tags to provide //! appropriate completions and auto-closing behavior. +// TODO: is this module in the right spot or even needed? + /// Tracks what closing characters are needed to complete a template tag. /// /// Used to determine whether the completion system needs to insert From d4b0397fd19a0f27de31e438b88ed3a8867cf009 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 3 Sep 2025 10:26:14 -0500 Subject: [PATCH 30/30] fix some documentation --- crates/djls-server/src/session.rs | 87 +++++++++++++------------------ 1 file changed, 36 insertions(+), 51 deletions(-) diff --git a/crates/djls-server/src/session.rs b/crates/djls-server/src/session.rs index be9224c..e0ffc3c 100644 --- a/crates/djls-server/src/session.rs +++ b/crates/djls-server/src/session.rs @@ -1,13 +1,13 @@ -//! # Salsa StorageHandle Pattern for LSP +//! # Salsa [`StorageHandle`] Pattern for LSP //! //! This module implements a thread-safe Salsa database wrapper for use with //! tower-lsp's async runtime. The key challenge is that tower-lsp requires //! `Send + Sync + 'static` bounds, but Salsa's `Storage` contains thread-local //! state and is not `Send`. //! -//! ## The Solution: StorageHandle +//! ## The Solution: [`StorageHandle`] //! -//! Salsa provides `StorageHandle` which IS `Send + Sync` because it contains +//! Salsa provides [`StorageHandle`] which IS `Send + Sync` because it contains //! no thread-local state. We store the handle and create `Storage`/`Database` //! instances on-demand. //! @@ -26,10 +26,12 @@ //! //! ## The Pattern //! -//! - **Reads**: Clone the handle freely (`with_db`) -//! - **Mutations**: Take exclusive ownership (`with_db_mut` via `take_db_handle_for_mutation`) +//! - **Reads**: Clone the handle freely ([`with_db`](Session::with_db)) +//! - **Mutations**: Take exclusive ownership ([`with_db_mut`](Session::with_db_mut) via [`take_db_handle_for_mutation`](Session::take_db_handle_for_mutation)) //! //! The explicit method names make the intent clear and prevent accidental misuse. +//! +//! [`StorageHandle`]: salsa::StorageHandle use std::path::Path; use std::path::PathBuf; @@ -52,12 +54,12 @@ use url::Url; /// LSP Session with thread-safe Salsa database access. /// -/// Uses Salsa's `StorageHandle` pattern to maintain `Send + Sync + 'static` +/// Uses Salsa's [`StorageHandle`] pattern to maintain `Send + Sync + 'static` /// compatibility required by tower-lsp. The handle can be safely shared /// across threads and async boundaries. /// /// See [this Salsa Zulip discussion](https://salsa.zulipchat.com/#narrow/channel/145099-Using-Salsa/topic/.E2.9C.94.20Advice.20on.20using.20salsa.20from.20Sync.20.2B.20Send.20context/with/495497515) -/// for more information about `StorageHandle`. +/// for more information about [`StorageHandle`]. /// /// ## Architecture /// @@ -69,11 +71,13 @@ use url::Url; /// /// When mutating Salsa inputs (like changing file revisions), we must ensure /// exclusive access to prevent race conditions. Salsa enforces this through -/// its `cancel_others()` mechanism, which waits for all `StorageHandle` clones +/// its `cancel_others()` mechanism, which waits for all [`StorageHandle`] clones /// to drop before allowing mutations. /// /// We use explicit methods (`take_db_handle_for_mutation`/`restore_db_handle`) /// to make this ownership transfer clear and prevent accidental deadlocks. +/// +/// [`StorageHandle`]: salsa::StorageHandle pub struct Session { /// The Django project configuration project: Option, @@ -85,27 +89,27 @@ pub struct Session { /// /// This implements Ruff's two-layer architecture where Layer 1 contains /// open document buffers that take precedence over disk files. The buffers - /// are shared between Session (which manages them) and WorkspaceFileSystem - /// (which reads from them). + /// are shared between Session (which manages them) and + /// [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) (which reads from them). /// /// Key properties: /// - Thread-safe via the Buffers abstraction - /// - Contains full TextDocument with content, version, and metadata + /// - Contains full [`TextDocument`](djls_workspace::TextDocument) with content, version, and metadata /// - Never becomes Salsa inputs - only intercepted at read time buffers: Buffers, /// File system abstraction with buffer interception /// - /// This WorkspaceFileSystem bridges Layer 1 (buffers) and Layer 2 (Salsa). - /// It intercepts FileSystem::read_to_string() calls to return buffer + /// This [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) bridges Layer 1 (buffers) and Layer 2 (Salsa). + /// It intercepts [`FileSystem::read_to_string()`](djls_workspace::FileSystem::read_to_string()) calls to return buffer /// content when available, falling back to disk otherwise. file_system: Arc, /// Shared file tracking across all Database instances /// /// This is the canonical Salsa pattern from the lazy-input example. - /// The DashMap provides O(1) lookups and is shared via Arc across - /// all Database instances created from StorageHandle. + /// The [`DashMap`] provides O(1) lookups and is shared via Arc across + /// all Database instances created from [`StorageHandle`](salsa::StorageHandle). files: Arc>, #[allow(dead_code)] @@ -113,11 +117,11 @@ pub struct Session { /// Layer 2: Thread-safe Salsa database handle for pure computation /// - /// where we're using the `StorageHandle` to create a thread-safe handle that can be + /// where we're using the [`StorageHandle`](salsa::StorageHandle) to create a thread-safe handle that can be /// shared between threads. /// - /// The database receives file content via the FileSystem trait, which - /// is intercepted by our LspFileSystem to provide overlay content. + /// The database receives file content via the [`FileSystem`](djls_workspace::FileSystem) trait, which + /// is intercepted by our [`WorkspaceFileSystem`](djls_workspace::WorkspaceFileSystem) to provide overlay content. /// This maintains proper separation between Layer 1 and Layer 2. db_handle: StorageHandle, } @@ -191,9 +195,13 @@ impl Session { self.settings = settings; } + // TODO: Explore an abstraction around [`salsa::StorageHandle`] and the following two methods + // to make it easy in the future to avoid deadlocks. For now, this is simpler and TBH may be + // all we ever need, but still.. might be a nice CYA for future me + /// Takes exclusive ownership of the database handle for mutation operations. /// - /// This method extracts the `StorageHandle` from the session, replacing it + /// This method extracts the [`StorageHandle`](salsa::StorageHandle) from the session, replacing it /// with a temporary placeholder. This ensures there's exactly one handle /// active during mutations, preventing deadlocks in Salsa's `cancel_others()`. /// @@ -205,8 +213,9 @@ impl Session { /// /// ## Panics /// - /// This is an internal method that should only be called by `with_db_mut`. - /// Multiple concurrent calls would panic when trying to take an already-taken handle. + /// This is an internal method that should only be called by + /// [`with_db_mut`](Session::with_db_mut). Multiple concurrent calls would panic when trying + /// to take an already-taken handle. fn take_db_handle_for_mutation(&mut self) -> StorageHandle { std::mem::replace(&mut self.db_handle, StorageHandle::new(None)) } @@ -290,7 +299,7 @@ impl Session { /// /// This method coordinates both layers: /// - Layer 1: Stores the document content in buffers - /// - Layer 2: Creates the SourceFile in Salsa (if path is resolvable) + /// - Layer 2: Creates the [`SourceFile`](djls_workspace::SourceFile) in Salsa (if path is resolvable) pub fn open_document(&mut self, url: &Url, document: TextDocument) { tracing::debug!("Opening document: {}", url); @@ -300,7 +309,7 @@ impl Session { // Layer 2: Create file and touch if it already exists // This is crucial: if the file was already read from disk, we need to // invalidate Salsa's cache so it re-reads through the buffer system - if let Some(path) = paths::url_to_path(&url) { + if let Some(path) = paths::url_to_path(url) { self.with_db_mut(|db| { // Check if file already exists (was previously read from disk) let already_exists = db.has_file(&path); @@ -389,15 +398,11 @@ impl Session { removed } - // ===== Safe Query API ===== - // These methods encapsulate all Salsa interactions, preventing the - // "mixed database instance" bug by never exposing SourceFile or Database. - /// Get the current content of a file (from overlay or disk). /// /// This is the safe way to read file content through the system. /// The file is created if it doesn't exist, and content is read - /// through the FileSystem abstraction (overlay first, then disk). + /// through the `FileSystem` abstraction (overlay first, then disk). pub fn file_content(&mut self, path: PathBuf) -> String { use djls_workspace::db::source_text; @@ -452,22 +457,17 @@ impl Default for Session { #[cfg(test)] mod tests { - use djls_workspace::LanguageId; - use super::*; + use djls_workspace::LanguageId; #[test] fn test_revision_invalidation_chain() { - use std::path::PathBuf; - let mut session = Session::default(); - // Create a test file path let path = PathBuf::from("/test/template.html"); let url = Url::parse("file:///test/template.html").unwrap(); // Open document with initial content - println!("**[test]** open document with initial content"); let document = TextDocument::new( "

Original Content

".to_string(), 1, @@ -475,50 +475,36 @@ mod tests { ); session.open_document(&url, document); - // Try to read content - this might be where it hangs - println!("**[test]** try to read content - this might be where it hangs"); let content1 = session.file_content(path.clone()); assert_eq!(content1, "

Original Content

"); // Update document with new content - println!("**[test]** Update document with new content"); let updated_document = TextDocument::new("

Updated Content

".to_string(), 2, LanguageId::Other); session.update_document(&url, updated_document); // Read content again (should get new overlay content due to invalidation) - println!( - "**[test]** Read content again (should get new overlay content due to invalidation)" - ); let content2 = session.file_content(path.clone()); assert_eq!(content2, "

Updated Content

"); assert_ne!(content1, content2); // Close document (removes overlay, bumps revision) - println!("**[test]** Close document (removes overlay, bumps revision)"); session.close_document(&url); // Read content again (should now read from disk, which returns empty for missing files) - println!( - "**[test]** Read content again (should now read from disk, which returns empty for missing files)" - ); let content3 = session.file_content(path.clone()); assert_eq!(content3, ""); // No file on disk, returns empty } #[test] fn test_with_db_mut_preserves_files() { - use std::path::PathBuf; - let mut session = Session::default(); - // Create multiple files let path1 = PathBuf::from("/test/file1.py"); let path2 = PathBuf::from("/test/file2.py"); - // Create files through safe API - session.file_content(path1.clone()); // Creates file1 - session.file_content(path2.clone()); // Creates file2 + session.file_content(path1.clone()); + session.file_content(path2.clone()); // Verify files are preserved across operations assert!(session.has_file(&path1)); @@ -532,7 +518,6 @@ mod tests { assert_eq!(content1, ""); assert_eq!(content2, ""); - // One more verification assert!(session.has_file(&path1)); assert!(session.has_file(&path2)); }