diff --git a/Cargo.lock b/Cargo.lock index 10a58c132b..0362914efa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4266,6 +4266,7 @@ dependencies = [ "ty_ide", "ty_project", "ty_python_semantic", + "ty_vendored", ] [[package]] @@ -4305,6 +4306,7 @@ version = "0.0.0" dependencies = [ "path-slash", "ruff_db", + "static_assertions", "walkdir", "zip", ] diff --git a/crates/ruff_db/src/system.rs b/crates/ruff_db/src/system.rs index 836010241f..e8b3062f9f 100644 --- a/crates/ruff_db/src/system.rs +++ b/crates/ruff_db/src/system.rs @@ -124,6 +124,11 @@ pub trait System: Debug { /// Returns `None` if no such convention exists for the system. fn user_config_directory(&self) -> Option; + /// Returns the directory path where cached files are stored. + /// + /// Returns `None` if no such convention exists for the system. + fn cache_dir(&self) -> Option; + /// Iterate over the contents of the directory at `path`. /// /// The returned iterator must have the following properties: @@ -186,6 +191,9 @@ pub trait System: Debug { Err(std::env::VarError::NotPresent) } + /// Returns a handle to a [`WritableSystem`] if this system is writeable. + fn as_writable(&self) -> Option<&dyn WritableSystem>; + fn as_any(&self) -> &dyn std::any::Any; fn as_any_mut(&mut self) -> &mut dyn std::any::Any; @@ -226,11 +234,52 @@ impl fmt::Display for CaseSensitivity { /// System trait for non-readonly systems. pub trait WritableSystem: System { + /// Creates a file at the given path. + /// + /// Returns an error if the file already exists. + fn create_new_file(&self, path: &SystemPath) -> Result<()>; + /// Writes the given content to the file at the given path. fn write_file(&self, path: &SystemPath, content: &str) -> Result<()>; /// Creates a directory at `path` as well as any intermediate directories. fn create_directory_all(&self, path: &SystemPath) -> Result<()>; + + /// Reads the provided file from the system cache, or creates the file if necessary. + /// + /// Returns `Ok(None)` if the system does not expose a suitable cache directory. + fn get_or_cache( + &self, + path: &SystemPath, + read_contents: &dyn Fn() -> Result, + ) -> Result> { + let Some(cache_dir) = self.cache_dir() else { + return Ok(None); + }; + + let cache_path = cache_dir.join(path); + + // The file has already been cached. + if self.is_file(&cache_path) { + return Ok(Some(cache_path)); + } + + // Read the file contents. + let contents = read_contents()?; + + // Create the parent directory. + self.create_directory_all(cache_path.parent().unwrap())?; + + // Create and write to the file on the system. + // + // Note that `create_new_file` will fail if the file has already been created. This + // ensures that only one thread/process ever attempts to write to it to avoid corrupting + // the cache. + self.create_new_file(&cache_path)?; + self.write_file(&cache_path, &contents)?; + + Ok(Some(cache_path)) + } } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/crates/ruff_db/src/system/memory_fs.rs b/crates/ruff_db/src/system/memory_fs.rs index 91587949a7..d5cbaa7d63 100644 --- a/crates/ruff_db/src/system/memory_fs.rs +++ b/crates/ruff_db/src/system/memory_fs.rs @@ -1,4 +1,5 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, btree_map}; +use std::io; use std::iter::FusedIterator; use std::sync::{Arc, RwLock, RwLockWriteGuard}; @@ -153,6 +154,26 @@ impl MemoryFileSystem { virtual_files.contains_key(&path.to_path_buf()) } + pub(crate) fn create_new_file(&self, path: &SystemPath) -> Result<()> { + let normalized = self.normalize_path(path); + + let mut by_path = self.inner.by_path.write().unwrap(); + match by_path.entry(normalized) { + btree_map::Entry::Vacant(entry) => { + entry.insert(Entry::File(File { + content: String::new(), + last_modified: file_time_now(), + })); + + Ok(()) + } + btree_map::Entry::Occupied(_) => Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "File already exists", + )), + } + } + /// Stores a new file in the file system. /// /// The operation overrides the content for an existing file with the same normalized `path`. @@ -278,14 +299,14 @@ impl MemoryFileSystem { let normalized = fs.normalize_path(path); match by_path.entry(normalized) { - std::collections::btree_map::Entry::Occupied(entry) => match entry.get() { + btree_map::Entry::Occupied(entry) => match entry.get() { Entry::File(_) => { entry.remove(); Ok(()) } Entry::Directory(_) => Err(is_a_directory()), }, - std::collections::btree_map::Entry::Vacant(_) => Err(not_found()), + btree_map::Entry::Vacant(_) => Err(not_found()), } } @@ -345,14 +366,14 @@ impl MemoryFileSystem { } match by_path.entry(normalized.clone()) { - std::collections::btree_map::Entry::Occupied(entry) => match entry.get() { + btree_map::Entry::Occupied(entry) => match entry.get() { Entry::Directory(_) => { entry.remove(); Ok(()) } Entry::File(_) => Err(not_a_directory()), }, - std::collections::btree_map::Entry::Vacant(_) => Err(not_found()), + btree_map::Entry::Vacant(_) => Err(not_found()), } } diff --git a/crates/ruff_db/src/system/os.rs b/crates/ruff_db/src/system/os.rs index 41b7bd0a1e..dd6a8eea99 100644 --- a/crates/ruff_db/src/system/os.rs +++ b/crates/ruff_db/src/system/os.rs @@ -160,6 +160,39 @@ impl System for OsSystem { None } + /// Returns an absolute cache directory on the system. + /// + /// On Linux and macOS, uses `$XDG_CACHE_HOME/ty` or `.cache/ty`. + /// On Windows, uses `C:\Users\User\AppData\Local\ty\cache`. + #[cfg(not(target_arch = "wasm32"))] + fn cache_dir(&self) -> Option { + use etcetera::BaseStrategy as _; + + let cache_dir = etcetera::base_strategy::choose_base_strategy() + .ok() + .map(|dirs| dirs.cache_dir().join("ty")) + .map(|cache_dir| { + if cfg!(windows) { + // On Windows, we append `cache` to the LocalAppData directory, i.e., prefer + // `C:\Users\User\AppData\Local\ty\cache` over `C:\Users\User\AppData\Local\ty`. + cache_dir.join("cache") + } else { + cache_dir + } + }) + .and_then(|path| SystemPathBuf::from_path_buf(path).ok()) + .unwrap_or_else(|| SystemPathBuf::from(".ty_cache")); + + Some(cache_dir) + } + + // TODO: Remove this feature gating once `ruff_wasm` no longer indirectly depends on `ruff_db` with the + // `os` feature enabled (via `ruff_workspace` -> `ruff_graph` -> `ruff_db`). + #[cfg(target_arch = "wasm32")] + fn cache_dir(&self) -> Option { + None + } + /// Creates a builder to recursively walk `path`. /// /// The walker ignores files according to [`ignore::WalkBuilder::standard_filters`] @@ -192,6 +225,10 @@ impl System for OsSystem { }) } + fn as_writable(&self) -> Option<&dyn WritableSystem> { + Some(self) + } + fn as_any(&self) -> &dyn Any { self } @@ -310,6 +347,10 @@ impl OsSystem { } impl WritableSystem for OsSystem { + fn create_new_file(&self, path: &SystemPath) -> Result<()> { + std::fs::File::create_new(path).map(drop) + } + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> { std::fs::write(path.as_std_path(), content) } diff --git a/crates/ruff_db/src/system/test.rs b/crates/ruff_db/src/system/test.rs index cfdf204bb0..f595aadca7 100644 --- a/crates/ruff_db/src/system/test.rs +++ b/crates/ruff_db/src/system/test.rs @@ -102,6 +102,10 @@ impl System for TestSystem { self.system().user_config_directory() } + fn cache_dir(&self) -> Option { + self.system().cache_dir() + } + fn read_directory<'a>( &'a self, path: &SystemPath, @@ -123,6 +127,10 @@ impl System for TestSystem { self.system().glob(pattern) } + fn as_writable(&self) -> Option<&dyn WritableSystem> { + Some(self) + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -149,6 +157,10 @@ impl Default for TestSystem { } impl WritableSystem for TestSystem { + fn create_new_file(&self, path: &SystemPath) -> Result<()> { + self.system().create_new_file(path) + } + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> { self.system().write_file(path, content) } @@ -335,6 +347,10 @@ impl System for InMemorySystem { self.user_config_directory.lock().unwrap().clone() } + fn cache_dir(&self) -> Option { + None + } + fn read_directory<'a>( &'a self, path: &SystemPath, @@ -357,6 +373,10 @@ impl System for InMemorySystem { Ok(Box::new(iterator)) } + fn as_writable(&self) -> Option<&dyn WritableSystem> { + Some(self) + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -377,6 +397,10 @@ impl System for InMemorySystem { } impl WritableSystem for InMemorySystem { + fn create_new_file(&self, path: &SystemPath) -> Result<()> { + self.memory_fs.create_new_file(path) + } + fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> { self.memory_fs.write_file(path, content) } diff --git a/crates/ruff_db/src/vendored.rs b/crates/ruff_db/src/vendored.rs index 7e8cd6e7cf..b1227af2a4 100644 --- a/crates/ruff_db/src/vendored.rs +++ b/crates/ruff_db/src/vendored.rs @@ -4,12 +4,12 @@ use std::fmt::{self, Debug}; use std::io::{self, Read, Write}; use std::sync::{Arc, Mutex, MutexGuard}; -use crate::file_revision::FileRevision; use zip::result::ZipResult; use zip::write::FileOptions; use zip::{CompressionMethod, ZipArchive, ZipWriter, read::ZipFile}; pub use self::path::{VendoredPath, VendoredPathBuf}; +use crate::file_revision::FileRevision; mod path; diff --git a/crates/ty_server/Cargo.toml b/crates/ty_server/Cargo.toml index 72c171051f..86c190799e 100644 --- a/crates/ty_server/Cargo.toml +++ b/crates/ty_server/Cargo.toml @@ -19,6 +19,7 @@ ruff_text_size = { workspace = true } ty_ide = { workspace = true } ty_project = { workspace = true } ty_python_semantic = { workspace = true } +ty_vendored = { workspace = true } anyhow = { workspace = true } crossbeam = { workspace = true } diff --git a/crates/ty_server/src/system.rs b/crates/ty_server/src/system.rs index e946029ff3..433e376847 100644 --- a/crates/ty_server/src/system.rs +++ b/crates/ty_server/src/system.rs @@ -8,7 +8,7 @@ use ruff_db::files::{File, FilePath}; use ruff_db::system::walk_directory::WalkDirectoryBuilder; use ruff_db::system::{ CaseSensitivity, DirectoryEntry, FileType, GlobError, Metadata, OsSystem, PatternError, Result, - System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf, + System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf, WritableSystem, }; use ruff_notebook::{Notebook, NotebookError}; use ty_python_semantic::Db; @@ -17,13 +17,29 @@ use crate::DocumentQuery; use crate::document::DocumentKey; use crate::session::index::Index; +/// Returns a [`Url`] for the given [`File`]. pub(crate) fn file_to_url(db: &dyn Db, file: File) -> Option { match file.path(db) { FilePath::System(system) => Url::from_file_path(system.as_std_path()).ok(), FilePath::SystemVirtual(path) => Url::parse(path.as_str()).ok(), - // TODO: Not yet supported, consider an approach similar to Sorbet's custom paths - // https://sorbet.org/docs/sorbet-uris - FilePath::Vendored(_) => None, + FilePath::Vendored(path) => { + let writable = db.system().as_writable()?; + + let system_path = SystemPathBuf::from(format!( + "vendored/typeshed/{}/{}", + // The vendored files are uniquely identified by the source commit. + ty_vendored::SOURCE_COMMIT, + path.as_str() + )); + + // Extract the vendored file onto the system. + let system_path = writable + .get_or_cache(&system_path, &|| db.vendored().read_to_string(path)) + .ok() + .flatten()?; + + Url::from_file_path(system_path.as_std_path()).ok() + } } } @@ -224,6 +240,10 @@ impl System for LSPSystem { self.os_system.user_config_directory() } + fn cache_dir(&self) -> Option { + self.os_system.cache_dir() + } + fn read_directory<'a>( &'a self, path: &SystemPath, @@ -245,6 +265,10 @@ impl System for LSPSystem { self.os_system.glob(pattern) } + fn as_writable(&self) -> Option<&dyn WritableSystem> { + self.os_system.as_writable() + } + fn as_any(&self) -> &dyn Any { self } diff --git a/crates/ty_test/src/db.rs b/crates/ty_test/src/db.rs index af5998ce18..76fff50e4e 100644 --- a/crates/ty_test/src/db.rs +++ b/crates/ty_test/src/db.rs @@ -226,6 +226,10 @@ impl System for MdtestSystem { self.as_system().user_config_directory() } + fn cache_dir(&self) -> Option { + self.as_system().cache_dir() + } + fn read_directory<'a>( &'a self, path: &SystemPath, @@ -253,6 +257,10 @@ impl System for MdtestSystem { .glob(self.normalize_path(SystemPath::new(pattern)).as_str()) } + fn as_writable(&self) -> Option<&dyn WritableSystem> { + Some(self) + } + fn as_any(&self) -> &dyn std::any::Any { self } @@ -263,6 +271,10 @@ impl System for MdtestSystem { } impl WritableSystem for MdtestSystem { + fn create_new_file(&self, path: &SystemPath) -> ruff_db::system::Result<()> { + self.as_system().create_new_file(&self.normalize_path(path)) + } + fn write_file(&self, path: &SystemPath, content: &str) -> ruff_db::system::Result<()> { self.as_system() .write_file(&self.normalize_path(path), content) diff --git a/crates/ty_vendored/Cargo.toml b/crates/ty_vendored/Cargo.toml index bd8f8e175b..0461b834c3 100644 --- a/crates/ty_vendored/Cargo.toml +++ b/crates/ty_vendored/Cargo.toml @@ -12,6 +12,7 @@ license = { workspace = true } [dependencies] ruff_db = { workspace = true } +static_assertions = { workspace = true } zip = { workspace = true } [build-dependencies] diff --git a/crates/ty_vendored/src/lib.rs b/crates/ty_vendored/src/lib.rs index d1a5816ec2..e6fd7e0a0b 100644 --- a/crates/ty_vendored/src/lib.rs +++ b/crates/ty_vendored/src/lib.rs @@ -1,6 +1,12 @@ use ruff_db::vendored::VendoredFileSystem; use std::sync::LazyLock; +/// The source commit of the vendored typeshed. +pub const SOURCE_COMMIT: &str = + include_str!("../../../crates/ty_vendored/vendor/typeshed/source_commit.txt").trim_ascii_end(); + +static_assertions::const_assert_eq!(SOURCE_COMMIT.len(), 40); + // The file path here is hardcoded in this crate's `build.rs` script. // Luckily this crate will fail to build if this file isn't available at build time. static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip")); diff --git a/crates/ty_wasm/src/lib.rs b/crates/ty_wasm/src/lib.rs index af4341cfa9..41d81fda68 100644 --- a/crates/ty_wasm/src/lib.rs +++ b/crates/ty_wasm/src/lib.rs @@ -8,7 +8,7 @@ use ruff_db::source::{line_index, source_text}; use ruff_db::system::walk_directory::WalkDirectoryBuilder; use ruff_db::system::{ CaseSensitivity, DirectoryEntry, GlobError, MemoryFileSystem, Metadata, PatternError, System, - SystemPath, SystemPathBuf, SystemVirtualPath, + SystemPath, SystemPathBuf, SystemVirtualPath, WritableSystem, }; use ruff_notebook::Notebook; use ruff_python_formatter::formatted_file; @@ -695,6 +695,10 @@ impl System for WasmSystem { None } + fn cache_dir(&self) -> Option { + None + } + fn read_directory<'a>( &'a self, path: &SystemPath, @@ -715,6 +719,10 @@ impl System for WasmSystem { Ok(Box::new(self.fs.glob(pattern)?)) } + fn as_writable(&self) -> Option<&dyn WritableSystem> { + None + } + fn as_any(&self) -> &dyn Any { self }