[ty] Support LSP go-to with vendored typeshed stubs (#19057)

## Summary

Extracts the vendored typeshed stubs lazily and caches them on the local
filesystem to support go-to in the LSP.

Resolves https://github.com/astral-sh/ty/issues/77.
This commit is contained in:
Ibraheem Ahmed 2025-07-02 07:58:58 -04:00 committed by GitHub
parent f7fc8fb084
commit ebc70a4002
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 200 additions and 11 deletions

2
Cargo.lock generated
View file

@ -4266,6 +4266,7 @@ dependencies = [
"ty_ide",
"ty_project",
"ty_python_semantic",
"ty_vendored",
]
[[package]]
@ -4305,6 +4306,7 @@ version = "0.0.0"
dependencies = [
"path-slash",
"ruff_db",
"static_assertions",
"walkdir",
"zip",
]

View file

@ -124,6 +124,11 @@ pub trait System: Debug {
/// Returns `None` if no such convention exists for the system.
fn user_config_directory(&self) -> Option<SystemPathBuf>;
/// Returns the directory path where cached files are stored.
///
/// Returns `None` if no such convention exists for the system.
fn cache_dir(&self) -> Option<SystemPathBuf>;
/// Iterate over the contents of the directory at `path`.
///
/// The returned iterator must have the following properties:
@ -186,6 +191,9 @@ pub trait System: Debug {
Err(std::env::VarError::NotPresent)
}
/// Returns a handle to a [`WritableSystem`] if this system is writeable.
fn as_writable(&self) -> Option<&dyn WritableSystem>;
fn as_any(&self) -> &dyn std::any::Any;
fn as_any_mut(&mut self) -> &mut dyn std::any::Any;
@ -226,11 +234,52 @@ impl fmt::Display for CaseSensitivity {
/// System trait for non-readonly systems.
pub trait WritableSystem: System {
/// Creates a file at the given path.
///
/// Returns an error if the file already exists.
fn create_new_file(&self, path: &SystemPath) -> Result<()>;
/// Writes the given content to the file at the given path.
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()>;
/// Creates a directory at `path` as well as any intermediate directories.
fn create_directory_all(&self, path: &SystemPath) -> Result<()>;
/// Reads the provided file from the system cache, or creates the file if necessary.
///
/// Returns `Ok(None)` if the system does not expose a suitable cache directory.
fn get_or_cache(
&self,
path: &SystemPath,
read_contents: &dyn Fn() -> Result<String>,
) -> Result<Option<SystemPathBuf>> {
let Some(cache_dir) = self.cache_dir() else {
return Ok(None);
};
let cache_path = cache_dir.join(path);
// The file has already been cached.
if self.is_file(&cache_path) {
return Ok(Some(cache_path));
}
// Read the file contents.
let contents = read_contents()?;
// Create the parent directory.
self.create_directory_all(cache_path.parent().unwrap())?;
// Create and write to the file on the system.
//
// Note that `create_new_file` will fail if the file has already been created. This
// ensures that only one thread/process ever attempts to write to it to avoid corrupting
// the cache.
self.create_new_file(&cache_path)?;
self.write_file(&cache_path, &contents)?;
Ok(Some(cache_path))
}
}
#[derive(Clone, Debug, Eq, PartialEq)]

View file

@ -1,4 +1,5 @@
use std::collections::BTreeMap;
use std::collections::{BTreeMap, btree_map};
use std::io;
use std::iter::FusedIterator;
use std::sync::{Arc, RwLock, RwLockWriteGuard};
@ -153,6 +154,26 @@ impl MemoryFileSystem {
virtual_files.contains_key(&path.to_path_buf())
}
pub(crate) fn create_new_file(&self, path: &SystemPath) -> Result<()> {
let normalized = self.normalize_path(path);
let mut by_path = self.inner.by_path.write().unwrap();
match by_path.entry(normalized) {
btree_map::Entry::Vacant(entry) => {
entry.insert(Entry::File(File {
content: String::new(),
last_modified: file_time_now(),
}));
Ok(())
}
btree_map::Entry::Occupied(_) => Err(io::Error::new(
io::ErrorKind::AlreadyExists,
"File already exists",
)),
}
}
/// Stores a new file in the file system.
///
/// The operation overrides the content for an existing file with the same normalized `path`.
@ -278,14 +299,14 @@ impl MemoryFileSystem {
let normalized = fs.normalize_path(path);
match by_path.entry(normalized) {
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
btree_map::Entry::Occupied(entry) => match entry.get() {
Entry::File(_) => {
entry.remove();
Ok(())
}
Entry::Directory(_) => Err(is_a_directory()),
},
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
btree_map::Entry::Vacant(_) => Err(not_found()),
}
}
@ -345,14 +366,14 @@ impl MemoryFileSystem {
}
match by_path.entry(normalized.clone()) {
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
btree_map::Entry::Occupied(entry) => match entry.get() {
Entry::Directory(_) => {
entry.remove();
Ok(())
}
Entry::File(_) => Err(not_a_directory()),
},
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
btree_map::Entry::Vacant(_) => Err(not_found()),
}
}

View file

@ -160,6 +160,39 @@ impl System for OsSystem {
None
}
/// Returns an absolute cache directory on the system.
///
/// On Linux and macOS, uses `$XDG_CACHE_HOME/ty` or `.cache/ty`.
/// On Windows, uses `C:\Users\User\AppData\Local\ty\cache`.
#[cfg(not(target_arch = "wasm32"))]
fn cache_dir(&self) -> Option<SystemPathBuf> {
use etcetera::BaseStrategy as _;
let cache_dir = etcetera::base_strategy::choose_base_strategy()
.ok()
.map(|dirs| dirs.cache_dir().join("ty"))
.map(|cache_dir| {
if cfg!(windows) {
// On Windows, we append `cache` to the LocalAppData directory, i.e., prefer
// `C:\Users\User\AppData\Local\ty\cache` over `C:\Users\User\AppData\Local\ty`.
cache_dir.join("cache")
} else {
cache_dir
}
})
.and_then(|path| SystemPathBuf::from_path_buf(path).ok())
.unwrap_or_else(|| SystemPathBuf::from(".ty_cache"));
Some(cache_dir)
}
// TODO: Remove this feature gating once `ruff_wasm` no longer indirectly depends on `ruff_db` with the
// `os` feature enabled (via `ruff_workspace` -> `ruff_graph` -> `ruff_db`).
#[cfg(target_arch = "wasm32")]
fn cache_dir(&self) -> Option<SystemPathBuf> {
None
}
/// Creates a builder to recursively walk `path`.
///
/// The walker ignores files according to [`ignore::WalkBuilder::standard_filters`]
@ -192,6 +225,10 @@ impl System for OsSystem {
})
}
fn as_writable(&self) -> Option<&dyn WritableSystem> {
Some(self)
}
fn as_any(&self) -> &dyn Any {
self
}
@ -310,6 +347,10 @@ impl OsSystem {
}
impl WritableSystem for OsSystem {
fn create_new_file(&self, path: &SystemPath) -> Result<()> {
std::fs::File::create_new(path).map(drop)
}
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
std::fs::write(path.as_std_path(), content)
}

View file

@ -102,6 +102,10 @@ impl System for TestSystem {
self.system().user_config_directory()
}
fn cache_dir(&self) -> Option<SystemPathBuf> {
self.system().cache_dir()
}
fn read_directory<'a>(
&'a self,
path: &SystemPath,
@ -123,6 +127,10 @@ impl System for TestSystem {
self.system().glob(pattern)
}
fn as_writable(&self) -> Option<&dyn WritableSystem> {
Some(self)
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
@ -149,6 +157,10 @@ impl Default for TestSystem {
}
impl WritableSystem for TestSystem {
fn create_new_file(&self, path: &SystemPath) -> Result<()> {
self.system().create_new_file(path)
}
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
self.system().write_file(path, content)
}
@ -335,6 +347,10 @@ impl System for InMemorySystem {
self.user_config_directory.lock().unwrap().clone()
}
fn cache_dir(&self) -> Option<SystemPathBuf> {
None
}
fn read_directory<'a>(
&'a self,
path: &SystemPath,
@ -357,6 +373,10 @@ impl System for InMemorySystem {
Ok(Box::new(iterator))
}
fn as_writable(&self) -> Option<&dyn WritableSystem> {
Some(self)
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
@ -377,6 +397,10 @@ impl System for InMemorySystem {
}
impl WritableSystem for InMemorySystem {
fn create_new_file(&self, path: &SystemPath) -> Result<()> {
self.memory_fs.create_new_file(path)
}
fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
self.memory_fs.write_file(path, content)
}

View file

@ -4,12 +4,12 @@ use std::fmt::{self, Debug};
use std::io::{self, Read, Write};
use std::sync::{Arc, Mutex, MutexGuard};
use crate::file_revision::FileRevision;
use zip::result::ZipResult;
use zip::write::FileOptions;
use zip::{CompressionMethod, ZipArchive, ZipWriter, read::ZipFile};
pub use self::path::{VendoredPath, VendoredPathBuf};
use crate::file_revision::FileRevision;
mod path;

View file

@ -19,6 +19,7 @@ ruff_text_size = { workspace = true }
ty_ide = { workspace = true }
ty_project = { workspace = true }
ty_python_semantic = { workspace = true }
ty_vendored = { workspace = true }
anyhow = { workspace = true }
crossbeam = { workspace = true }

View file

@ -8,7 +8,7 @@ use ruff_db::files::{File, FilePath};
use ruff_db::system::walk_directory::WalkDirectoryBuilder;
use ruff_db::system::{
CaseSensitivity, DirectoryEntry, FileType, GlobError, Metadata, OsSystem, PatternError, Result,
System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf,
System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf, WritableSystem,
};
use ruff_notebook::{Notebook, NotebookError};
use ty_python_semantic::Db;
@ -17,13 +17,29 @@ use crate::DocumentQuery;
use crate::document::DocumentKey;
use crate::session::index::Index;
/// Returns a [`Url`] for the given [`File`].
pub(crate) fn file_to_url(db: &dyn Db, file: File) -> Option<Url> {
match file.path(db) {
FilePath::System(system) => Url::from_file_path(system.as_std_path()).ok(),
FilePath::SystemVirtual(path) => Url::parse(path.as_str()).ok(),
// TODO: Not yet supported, consider an approach similar to Sorbet's custom paths
// https://sorbet.org/docs/sorbet-uris
FilePath::Vendored(_) => None,
FilePath::Vendored(path) => {
let writable = db.system().as_writable()?;
let system_path = SystemPathBuf::from(format!(
"vendored/typeshed/{}/{}",
// The vendored files are uniquely identified by the source commit.
ty_vendored::SOURCE_COMMIT,
path.as_str()
));
// Extract the vendored file onto the system.
let system_path = writable
.get_or_cache(&system_path, &|| db.vendored().read_to_string(path))
.ok()
.flatten()?;
Url::from_file_path(system_path.as_std_path()).ok()
}
}
}
@ -224,6 +240,10 @@ impl System for LSPSystem {
self.os_system.user_config_directory()
}
fn cache_dir(&self) -> Option<SystemPathBuf> {
self.os_system.cache_dir()
}
fn read_directory<'a>(
&'a self,
path: &SystemPath,
@ -245,6 +265,10 @@ impl System for LSPSystem {
self.os_system.glob(pattern)
}
fn as_writable(&self) -> Option<&dyn WritableSystem> {
self.os_system.as_writable()
}
fn as_any(&self) -> &dyn Any {
self
}

View file

@ -226,6 +226,10 @@ impl System for MdtestSystem {
self.as_system().user_config_directory()
}
fn cache_dir(&self) -> Option<SystemPathBuf> {
self.as_system().cache_dir()
}
fn read_directory<'a>(
&'a self,
path: &SystemPath,
@ -253,6 +257,10 @@ impl System for MdtestSystem {
.glob(self.normalize_path(SystemPath::new(pattern)).as_str())
}
fn as_writable(&self) -> Option<&dyn WritableSystem> {
Some(self)
}
fn as_any(&self) -> &dyn std::any::Any {
self
}
@ -263,6 +271,10 @@ impl System for MdtestSystem {
}
impl WritableSystem for MdtestSystem {
fn create_new_file(&self, path: &SystemPath) -> ruff_db::system::Result<()> {
self.as_system().create_new_file(&self.normalize_path(path))
}
fn write_file(&self, path: &SystemPath, content: &str) -> ruff_db::system::Result<()> {
self.as_system()
.write_file(&self.normalize_path(path), content)

View file

@ -12,6 +12,7 @@ license = { workspace = true }
[dependencies]
ruff_db = { workspace = true }
static_assertions = { workspace = true }
zip = { workspace = true }
[build-dependencies]

View file

@ -1,6 +1,12 @@
use ruff_db::vendored::VendoredFileSystem;
use std::sync::LazyLock;
/// The source commit of the vendored typeshed.
pub const SOURCE_COMMIT: &str =
include_str!("../../../crates/ty_vendored/vendor/typeshed/source_commit.txt").trim_ascii_end();
static_assertions::const_assert_eq!(SOURCE_COMMIT.len(), 40);
// The file path here is hardcoded in this crate's `build.rs` script.
// Luckily this crate will fail to build if this file isn't available at build time.
static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip"));

View file

@ -8,7 +8,7 @@ use ruff_db::source::{line_index, source_text};
use ruff_db::system::walk_directory::WalkDirectoryBuilder;
use ruff_db::system::{
CaseSensitivity, DirectoryEntry, GlobError, MemoryFileSystem, Metadata, PatternError, System,
SystemPath, SystemPathBuf, SystemVirtualPath,
SystemPath, SystemPathBuf, SystemVirtualPath, WritableSystem,
};
use ruff_notebook::Notebook;
use ruff_python_formatter::formatted_file;
@ -695,6 +695,10 @@ impl System for WasmSystem {
None
}
fn cache_dir(&self) -> Option<SystemPathBuf> {
None
}
fn read_directory<'a>(
&'a self,
path: &SystemPath,
@ -715,6 +719,10 @@ impl System for WasmSystem {
Ok(Box::new(self.fs.glob(pattern)?))
}
fn as_writable(&self) -> Option<&dyn WritableSystem> {
None
}
fn as_any(&self) -> &dyn Any {
self
}