mirror of
https://github.com/astral-sh/ruff.git
synced 2025-07-07 13:15:06 +00:00
[ty] Support LSP go-to with vendored typeshed stubs (#19057)
## Summary Extracts the vendored typeshed stubs lazily and caches them on the local filesystem to support go-to in the LSP. Resolves https://github.com/astral-sh/ty/issues/77.
This commit is contained in:
parent
f7fc8fb084
commit
ebc70a4002
12 changed files with 200 additions and 11 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
@ -4266,6 +4266,7 @@ dependencies = [
|
|||
"ty_ide",
|
||||
"ty_project",
|
||||
"ty_python_semantic",
|
||||
"ty_vendored",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
@ -4305,6 +4306,7 @@ version = "0.0.0"
|
|||
dependencies = [
|
||||
"path-slash",
|
||||
"ruff_db",
|
||||
"static_assertions",
|
||||
"walkdir",
|
||||
"zip",
|
||||
]
|
||||
|
|
|
@ -124,6 +124,11 @@ pub trait System: Debug {
|
|||
/// Returns `None` if no such convention exists for the system.
|
||||
fn user_config_directory(&self) -> Option<SystemPathBuf>;
|
||||
|
||||
/// Returns the directory path where cached files are stored.
|
||||
///
|
||||
/// Returns `None` if no such convention exists for the system.
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf>;
|
||||
|
||||
/// Iterate over the contents of the directory at `path`.
|
||||
///
|
||||
/// The returned iterator must have the following properties:
|
||||
|
@ -186,6 +191,9 @@ pub trait System: Debug {
|
|||
Err(std::env::VarError::NotPresent)
|
||||
}
|
||||
|
||||
/// Returns a handle to a [`WritableSystem`] if this system is writeable.
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem>;
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any;
|
||||
|
||||
fn as_any_mut(&mut self) -> &mut dyn std::any::Any;
|
||||
|
@ -226,11 +234,52 @@ impl fmt::Display for CaseSensitivity {
|
|||
|
||||
/// System trait for non-readonly systems.
|
||||
pub trait WritableSystem: System {
|
||||
/// Creates a file at the given path.
|
||||
///
|
||||
/// Returns an error if the file already exists.
|
||||
fn create_new_file(&self, path: &SystemPath) -> Result<()>;
|
||||
|
||||
/// Writes the given content to the file at the given path.
|
||||
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()>;
|
||||
|
||||
/// Creates a directory at `path` as well as any intermediate directories.
|
||||
fn create_directory_all(&self, path: &SystemPath) -> Result<()>;
|
||||
|
||||
/// Reads the provided file from the system cache, or creates the file if necessary.
|
||||
///
|
||||
/// Returns `Ok(None)` if the system does not expose a suitable cache directory.
|
||||
fn get_or_cache(
|
||||
&self,
|
||||
path: &SystemPath,
|
||||
read_contents: &dyn Fn() -> Result<String>,
|
||||
) -> Result<Option<SystemPathBuf>> {
|
||||
let Some(cache_dir) = self.cache_dir() else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let cache_path = cache_dir.join(path);
|
||||
|
||||
// The file has already been cached.
|
||||
if self.is_file(&cache_path) {
|
||||
return Ok(Some(cache_path));
|
||||
}
|
||||
|
||||
// Read the file contents.
|
||||
let contents = read_contents()?;
|
||||
|
||||
// Create the parent directory.
|
||||
self.create_directory_all(cache_path.parent().unwrap())?;
|
||||
|
||||
// Create and write to the file on the system.
|
||||
//
|
||||
// Note that `create_new_file` will fail if the file has already been created. This
|
||||
// ensures that only one thread/process ever attempts to write to it to avoid corrupting
|
||||
// the cache.
|
||||
self.create_new_file(&cache_path)?;
|
||||
self.write_file(&cache_path, &contents)?;
|
||||
|
||||
Ok(Some(cache_path))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, btree_map};
|
||||
use std::io;
|
||||
use std::iter::FusedIterator;
|
||||
use std::sync::{Arc, RwLock, RwLockWriteGuard};
|
||||
|
||||
|
@ -153,6 +154,26 @@ impl MemoryFileSystem {
|
|||
virtual_files.contains_key(&path.to_path_buf())
|
||||
}
|
||||
|
||||
pub(crate) fn create_new_file(&self, path: &SystemPath) -> Result<()> {
|
||||
let normalized = self.normalize_path(path);
|
||||
|
||||
let mut by_path = self.inner.by_path.write().unwrap();
|
||||
match by_path.entry(normalized) {
|
||||
btree_map::Entry::Vacant(entry) => {
|
||||
entry.insert(Entry::File(File {
|
||||
content: String::new(),
|
||||
last_modified: file_time_now(),
|
||||
}));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
btree_map::Entry::Occupied(_) => Err(io::Error::new(
|
||||
io::ErrorKind::AlreadyExists,
|
||||
"File already exists",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores a new file in the file system.
|
||||
///
|
||||
/// The operation overrides the content for an existing file with the same normalized `path`.
|
||||
|
@ -278,14 +299,14 @@ impl MemoryFileSystem {
|
|||
let normalized = fs.normalize_path(path);
|
||||
|
||||
match by_path.entry(normalized) {
|
||||
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
|
||||
btree_map::Entry::Occupied(entry) => match entry.get() {
|
||||
Entry::File(_) => {
|
||||
entry.remove();
|
||||
Ok(())
|
||||
}
|
||||
Entry::Directory(_) => Err(is_a_directory()),
|
||||
},
|
||||
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
|
||||
btree_map::Entry::Vacant(_) => Err(not_found()),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -345,14 +366,14 @@ impl MemoryFileSystem {
|
|||
}
|
||||
|
||||
match by_path.entry(normalized.clone()) {
|
||||
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
|
||||
btree_map::Entry::Occupied(entry) => match entry.get() {
|
||||
Entry::Directory(_) => {
|
||||
entry.remove();
|
||||
Ok(())
|
||||
}
|
||||
Entry::File(_) => Err(not_a_directory()),
|
||||
},
|
||||
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
|
||||
btree_map::Entry::Vacant(_) => Err(not_found()),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -160,6 +160,39 @@ impl System for OsSystem {
|
|||
None
|
||||
}
|
||||
|
||||
/// Returns an absolute cache directory on the system.
|
||||
///
|
||||
/// On Linux and macOS, uses `$XDG_CACHE_HOME/ty` or `.cache/ty`.
|
||||
/// On Windows, uses `C:\Users\User\AppData\Local\ty\cache`.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
use etcetera::BaseStrategy as _;
|
||||
|
||||
let cache_dir = etcetera::base_strategy::choose_base_strategy()
|
||||
.ok()
|
||||
.map(|dirs| dirs.cache_dir().join("ty"))
|
||||
.map(|cache_dir| {
|
||||
if cfg!(windows) {
|
||||
// On Windows, we append `cache` to the LocalAppData directory, i.e., prefer
|
||||
// `C:\Users\User\AppData\Local\ty\cache` over `C:\Users\User\AppData\Local\ty`.
|
||||
cache_dir.join("cache")
|
||||
} else {
|
||||
cache_dir
|
||||
}
|
||||
})
|
||||
.and_then(|path| SystemPathBuf::from_path_buf(path).ok())
|
||||
.unwrap_or_else(|| SystemPathBuf::from(".ty_cache"));
|
||||
|
||||
Some(cache_dir)
|
||||
}
|
||||
|
||||
// TODO: Remove this feature gating once `ruff_wasm` no longer indirectly depends on `ruff_db` with the
|
||||
// `os` feature enabled (via `ruff_workspace` -> `ruff_graph` -> `ruff_db`).
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Creates a builder to recursively walk `path`.
|
||||
///
|
||||
/// The walker ignores files according to [`ignore::WalkBuilder::standard_filters`]
|
||||
|
@ -192,6 +225,10 @@ impl System for OsSystem {
|
|||
})
|
||||
}
|
||||
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem> {
|
||||
Some(self)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
@ -310,6 +347,10 @@ impl OsSystem {
|
|||
}
|
||||
|
||||
impl WritableSystem for OsSystem {
|
||||
fn create_new_file(&self, path: &SystemPath) -> Result<()> {
|
||||
std::fs::File::create_new(path).map(drop)
|
||||
}
|
||||
|
||||
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
|
||||
std::fs::write(path.as_std_path(), content)
|
||||
}
|
||||
|
|
|
@ -102,6 +102,10 @@ impl System for TestSystem {
|
|||
self.system().user_config_directory()
|
||||
}
|
||||
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
self.system().cache_dir()
|
||||
}
|
||||
|
||||
fn read_directory<'a>(
|
||||
&'a self,
|
||||
path: &SystemPath,
|
||||
|
@ -123,6 +127,10 @@ impl System for TestSystem {
|
|||
self.system().glob(pattern)
|
||||
}
|
||||
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem> {
|
||||
Some(self)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
@ -149,6 +157,10 @@ impl Default for TestSystem {
|
|||
}
|
||||
|
||||
impl WritableSystem for TestSystem {
|
||||
fn create_new_file(&self, path: &SystemPath) -> Result<()> {
|
||||
self.system().create_new_file(path)
|
||||
}
|
||||
|
||||
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
|
||||
self.system().write_file(path, content)
|
||||
}
|
||||
|
@ -335,6 +347,10 @@ impl System for InMemorySystem {
|
|||
self.user_config_directory.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
fn read_directory<'a>(
|
||||
&'a self,
|
||||
path: &SystemPath,
|
||||
|
@ -357,6 +373,10 @@ impl System for InMemorySystem {
|
|||
Ok(Box::new(iterator))
|
||||
}
|
||||
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem> {
|
||||
Some(self)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
@ -377,6 +397,10 @@ impl System for InMemorySystem {
|
|||
}
|
||||
|
||||
impl WritableSystem for InMemorySystem {
|
||||
fn create_new_file(&self, path: &SystemPath) -> Result<()> {
|
||||
self.memory_fs.create_new_file(path)
|
||||
}
|
||||
|
||||
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
|
||||
self.memory_fs.write_file(path, content)
|
||||
}
|
||||
|
|
|
@ -4,12 +4,12 @@ use std::fmt::{self, Debug};
|
|||
use std::io::{self, Read, Write};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
|
||||
use crate::file_revision::FileRevision;
|
||||
use zip::result::ZipResult;
|
||||
use zip::write::FileOptions;
|
||||
use zip::{CompressionMethod, ZipArchive, ZipWriter, read::ZipFile};
|
||||
|
||||
pub use self::path::{VendoredPath, VendoredPathBuf};
|
||||
use crate::file_revision::FileRevision;
|
||||
|
||||
mod path;
|
||||
|
||||
|
|
|
@ -19,6 +19,7 @@ ruff_text_size = { workspace = true }
|
|||
ty_ide = { workspace = true }
|
||||
ty_project = { workspace = true }
|
||||
ty_python_semantic = { workspace = true }
|
||||
ty_vendored = { workspace = true }
|
||||
|
||||
anyhow = { workspace = true }
|
||||
crossbeam = { workspace = true }
|
||||
|
|
|
@ -8,7 +8,7 @@ use ruff_db::files::{File, FilePath};
|
|||
use ruff_db::system::walk_directory::WalkDirectoryBuilder;
|
||||
use ruff_db::system::{
|
||||
CaseSensitivity, DirectoryEntry, FileType, GlobError, Metadata, OsSystem, PatternError, Result,
|
||||
System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf,
|
||||
System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf, WritableSystem,
|
||||
};
|
||||
use ruff_notebook::{Notebook, NotebookError};
|
||||
use ty_python_semantic::Db;
|
||||
|
@ -17,13 +17,29 @@ use crate::DocumentQuery;
|
|||
use crate::document::DocumentKey;
|
||||
use crate::session::index::Index;
|
||||
|
||||
/// Returns a [`Url`] for the given [`File`].
|
||||
pub(crate) fn file_to_url(db: &dyn Db, file: File) -> Option<Url> {
|
||||
match file.path(db) {
|
||||
FilePath::System(system) => Url::from_file_path(system.as_std_path()).ok(),
|
||||
FilePath::SystemVirtual(path) => Url::parse(path.as_str()).ok(),
|
||||
// TODO: Not yet supported, consider an approach similar to Sorbet's custom paths
|
||||
// https://sorbet.org/docs/sorbet-uris
|
||||
FilePath::Vendored(_) => None,
|
||||
FilePath::Vendored(path) => {
|
||||
let writable = db.system().as_writable()?;
|
||||
|
||||
let system_path = SystemPathBuf::from(format!(
|
||||
"vendored/typeshed/{}/{}",
|
||||
// The vendored files are uniquely identified by the source commit.
|
||||
ty_vendored::SOURCE_COMMIT,
|
||||
path.as_str()
|
||||
));
|
||||
|
||||
// Extract the vendored file onto the system.
|
||||
let system_path = writable
|
||||
.get_or_cache(&system_path, &|| db.vendored().read_to_string(path))
|
||||
.ok()
|
||||
.flatten()?;
|
||||
|
||||
Url::from_file_path(system_path.as_std_path()).ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -224,6 +240,10 @@ impl System for LSPSystem {
|
|||
self.os_system.user_config_directory()
|
||||
}
|
||||
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
self.os_system.cache_dir()
|
||||
}
|
||||
|
||||
fn read_directory<'a>(
|
||||
&'a self,
|
||||
path: &SystemPath,
|
||||
|
@ -245,6 +265,10 @@ impl System for LSPSystem {
|
|||
self.os_system.glob(pattern)
|
||||
}
|
||||
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem> {
|
||||
self.os_system.as_writable()
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
|
|
@ -226,6 +226,10 @@ impl System for MdtestSystem {
|
|||
self.as_system().user_config_directory()
|
||||
}
|
||||
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
self.as_system().cache_dir()
|
||||
}
|
||||
|
||||
fn read_directory<'a>(
|
||||
&'a self,
|
||||
path: &SystemPath,
|
||||
|
@ -253,6 +257,10 @@ impl System for MdtestSystem {
|
|||
.glob(self.normalize_path(SystemPath::new(pattern)).as_str())
|
||||
}
|
||||
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem> {
|
||||
Some(self)
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn std::any::Any {
|
||||
self
|
||||
}
|
||||
|
@ -263,6 +271,10 @@ impl System for MdtestSystem {
|
|||
}
|
||||
|
||||
impl WritableSystem for MdtestSystem {
|
||||
fn create_new_file(&self, path: &SystemPath) -> ruff_db::system::Result<()> {
|
||||
self.as_system().create_new_file(&self.normalize_path(path))
|
||||
}
|
||||
|
||||
fn write_file(&self, path: &SystemPath, content: &str) -> ruff_db::system::Result<()> {
|
||||
self.as_system()
|
||||
.write_file(&self.normalize_path(path), content)
|
||||
|
|
|
@ -12,6 +12,7 @@ license = { workspace = true }
|
|||
|
||||
[dependencies]
|
||||
ruff_db = { workspace = true }
|
||||
static_assertions = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
|
|
|
@ -1,6 +1,12 @@
|
|||
use ruff_db::vendored::VendoredFileSystem;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// The source commit of the vendored typeshed.
|
||||
pub const SOURCE_COMMIT: &str =
|
||||
include_str!("../../../crates/ty_vendored/vendor/typeshed/source_commit.txt").trim_ascii_end();
|
||||
|
||||
static_assertions::const_assert_eq!(SOURCE_COMMIT.len(), 40);
|
||||
|
||||
// The file path here is hardcoded in this crate's `build.rs` script.
|
||||
// Luckily this crate will fail to build if this file isn't available at build time.
|
||||
static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip"));
|
||||
|
|
|
@ -8,7 +8,7 @@ use ruff_db::source::{line_index, source_text};
|
|||
use ruff_db::system::walk_directory::WalkDirectoryBuilder;
|
||||
use ruff_db::system::{
|
||||
CaseSensitivity, DirectoryEntry, GlobError, MemoryFileSystem, Metadata, PatternError, System,
|
||||
SystemPath, SystemPathBuf, SystemVirtualPath,
|
||||
SystemPath, SystemPathBuf, SystemVirtualPath, WritableSystem,
|
||||
};
|
||||
use ruff_notebook::Notebook;
|
||||
use ruff_python_formatter::formatted_file;
|
||||
|
@ -695,6 +695,10 @@ impl System for WasmSystem {
|
|||
None
|
||||
}
|
||||
|
||||
fn cache_dir(&self) -> Option<SystemPathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
fn read_directory<'a>(
|
||||
&'a self,
|
||||
path: &SystemPath,
|
||||
|
@ -715,6 +719,10 @@ impl System for WasmSystem {
|
|||
Ok(Box::new(self.fs.glob(pattern)?))
|
||||
}
|
||||
|
||||
fn as_writable(&self) -> Option<&dyn WritableSystem> {
|
||||
None
|
||||
}
|
||||
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue