[ty] Support LSP go-to with vendored typeshed stubs (#19057)

## Summary

Extracts the vendored typeshed stubs lazily and caches them on the local
filesystem to support go-to in the LSP.

Resolves https://github.com/astral-sh/ty/issues/77.

Cargo.lock

@@ -4266,6 +4266,7 @@ dependencies = [
  "ty_ide",
  "ty_project",
  "ty_python_semantic",
+ "ty_vendored",
 ]
 
 [[package]]
@@ -4305,6 +4306,7 @@ version = "0.0.0"
 dependencies = [
  "path-slash",
  "ruff_db",
+ "static_assertions",
  "walkdir",
  "zip",
 ]

crates/ruff_db/src/system.rs

@@ -124,6 +124,11 @@ pub trait System: Debug {
     /// Returns `None` if no such convention exists for the system.
     fn user_config_directory(&self) -> Option<SystemPathBuf>;
 
+    /// Returns the directory path where cached files are stored.
+    ///
+    /// Returns `None` if no such convention exists for the system.
+    fn cache_dir(&self) -> Option<SystemPathBuf>;
+
     /// Iterate over the contents of the directory at `path`.
     ///
     /// The returned iterator must have the following properties:
@@ -186,6 +191,9 @@ pub trait System: Debug {
         Err(std::env::VarError::NotPresent)
     }
 
+    /// Returns a handle to a [`WritableSystem`] if this system is writeable.
+    fn as_writable(&self) -> Option<&dyn WritableSystem>;
+
     fn as_any(&self) -> &dyn std::any::Any;
 
     fn as_any_mut(&mut self) -> &mut dyn std::any::Any;
@@ -226,11 +234,52 @@ impl fmt::Display for CaseSensitivity {
 
 /// System trait for non-readonly systems.
 pub trait WritableSystem: System {
+    /// Creates a file at the given path.
+    ///
+    /// Returns an error if the file already exists.
+    fn create_new_file(&self, path: &SystemPath) -> Result<()>;
+
     /// Writes the given content to the file at the given path.
     fn write_file(&self, path: &SystemPath, content: &str) -> Result<()>;
 
     /// Creates a directory at `path` as well as any intermediate directories.
     fn create_directory_all(&self, path: &SystemPath) -> Result<()>;
+
+    /// Reads the provided file from the system cache, or creates the file if necessary.
+    ///
+    /// Returns `Ok(None)` if the system does not expose a suitable cache directory.
+    fn get_or_cache(
+        &self,
+        path: &SystemPath,
+        read_contents: &dyn Fn() -> Result<String>,
+    ) -> Result<Option<SystemPathBuf>> {
+        let Some(cache_dir) = self.cache_dir() else {
+            return Ok(None);
+        };
+
+        let cache_path = cache_dir.join(path);
+
+        // The file has already been cached.
+        if self.is_file(&cache_path) {
+            return Ok(Some(cache_path));
+        }
+
+        // Read the file contents.
+        let contents = read_contents()?;
+
+        // Create the parent directory.
+        self.create_directory_all(cache_path.parent().unwrap())?;
+
+        // Create and write to the file on the system.
+        //
+        // Note that `create_new_file` will fail if the file has already been created. This
+        // ensures that only one thread/process ever attempts to write to it to avoid corrupting
+        // the cache.
+        self.create_new_file(&cache_path)?;
+        self.write_file(&cache_path, &contents)?;
+
+        Ok(Some(cache_path))
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]

crates/ruff_db/src/system/memory_fs.rs

@@ -1,4 +1,5 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, btree_map};
+use std::io;
 use std::iter::FusedIterator;
 use std::sync::{Arc, RwLock, RwLockWriteGuard};
 
@@ -153,6 +154,26 @@ impl MemoryFileSystem {
         virtual_files.contains_key(&path.to_path_buf())
     }
 
+    pub(crate) fn create_new_file(&self, path: &SystemPath) -> Result<()> {
+        let normalized = self.normalize_path(path);
+
+        let mut by_path = self.inner.by_path.write().unwrap();
+        match by_path.entry(normalized) {
+            btree_map::Entry::Vacant(entry) => {
+                entry.insert(Entry::File(File {
+                    content: String::new(),
+                    last_modified: file_time_now(),
+                }));
+
+                Ok(())
+            }
+            btree_map::Entry::Occupied(_) => Err(io::Error::new(
+                io::ErrorKind::AlreadyExists,
+                "File already exists",
+            )),
+        }
+    }
+
     /// Stores a new file in the file system.
     ///
     /// The operation overrides the content for an existing file with the same normalized `path`.
@@ -278,14 +299,14 @@ impl MemoryFileSystem {
             let normalized = fs.normalize_path(path);
 
             match by_path.entry(normalized) {
-                std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
+                btree_map::Entry::Occupied(entry) => match entry.get() {
                     Entry::File(_) => {
                         entry.remove();
                         Ok(())
                     }
                     Entry::Directory(_) => Err(is_a_directory()),
                 },
-                std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
+                btree_map::Entry::Vacant(_) => Err(not_found()),
             }
         }
 
@@ -345,14 +366,14 @@ impl MemoryFileSystem {
             }
 
             match by_path.entry(normalized.clone()) {
-                std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
+                btree_map::Entry::Occupied(entry) => match entry.get() {
                     Entry::Directory(_) => {
                         entry.remove();
                         Ok(())
                     }
                     Entry::File(_) => Err(not_a_directory()),
                 },
-                std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
+                btree_map::Entry::Vacant(_) => Err(not_found()),
             }
         }
 

crates/ruff_db/src/system/os.rs

@@ -160,6 +160,39 @@ impl System for OsSystem {
         None
     }
 
+    /// Returns an absolute cache directory on the system.
+    ///
+    /// On Linux and macOS, uses `$XDG_CACHE_HOME/ty` or `.cache/ty`.
+    /// On Windows, uses `C:\Users\User\AppData\Local\ty\cache`.
+    #[cfg(not(target_arch = "wasm32"))]
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        use etcetera::BaseStrategy as _;
+
+        let cache_dir = etcetera::base_strategy::choose_base_strategy()
+            .ok()
+            .map(|dirs| dirs.cache_dir().join("ty"))
+            .map(|cache_dir| {
+                if cfg!(windows) {
+                    // On Windows, we append `cache` to the LocalAppData directory, i.e., prefer
+                    // `C:\Users\User\AppData\Local\ty\cache` over `C:\Users\User\AppData\Local\ty`.
+                    cache_dir.join("cache")
+                } else {
+                    cache_dir
+                }
+            })
+            .and_then(|path| SystemPathBuf::from_path_buf(path).ok())
+            .unwrap_or_else(|| SystemPathBuf::from(".ty_cache"));
+
+        Some(cache_dir)
+    }
+
+    // TODO: Remove this feature gating once `ruff_wasm` no longer indirectly depends on `ruff_db` with the
+    //   `os` feature enabled (via `ruff_workspace` -> `ruff_graph` -> `ruff_db`).
+    #[cfg(target_arch = "wasm32")]
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        None
+    }
+
     /// Creates a builder to recursively walk `path`.
     ///
     /// The walker ignores files according to [`ignore::WalkBuilder::standard_filters`]
@@ -192,6 +225,10 @@ impl System for OsSystem {
         })
     }
 
+    fn as_writable(&self) -> Option<&dyn WritableSystem> {
+        Some(self)
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }
@@ -310,6 +347,10 @@ impl OsSystem {
 }
 
 impl WritableSystem for OsSystem {
+    fn create_new_file(&self, path: &SystemPath) -> Result<()> {
+        std::fs::File::create_new(path).map(drop)
+    }
+
     fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
         std::fs::write(path.as_std_path(), content)
     }

crates/ruff_db/src/system/test.rs

@@ -102,6 +102,10 @@ impl System for TestSystem {
         self.system().user_config_directory()
     }
 
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        self.system().cache_dir()
+    }
+
     fn read_directory<'a>(
         &'a self,
         path: &SystemPath,
@@ -123,6 +127,10 @@ impl System for TestSystem {
         self.system().glob(pattern)
     }
 
+    fn as_writable(&self) -> Option<&dyn WritableSystem> {
+        Some(self)
+    }
+
     fn as_any(&self) -> &dyn std::any::Any {
         self
     }
@@ -149,6 +157,10 @@ impl Default for TestSystem {
 }
 
 impl WritableSystem for TestSystem {
+    fn create_new_file(&self, path: &SystemPath) -> Result<()> {
+        self.system().create_new_file(path)
+    }
+
     fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
         self.system().write_file(path, content)
     }
@@ -335,6 +347,10 @@ impl System for InMemorySystem {
         self.user_config_directory.lock().unwrap().clone()
     }
 
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        None
+    }
+
     fn read_directory<'a>(
         &'a self,
         path: &SystemPath,
@@ -357,6 +373,10 @@ impl System for InMemorySystem {
         Ok(Box::new(iterator))
     }
 
+    fn as_writable(&self) -> Option<&dyn WritableSystem> {
+        Some(self)
+    }
+
     fn as_any(&self) -> &dyn std::any::Any {
         self
     }
@@ -377,6 +397,10 @@ impl System for InMemorySystem {
 }
 
 impl WritableSystem for InMemorySystem {
+    fn create_new_file(&self, path: &SystemPath) -> Result<()> {
+        self.memory_fs.create_new_file(path)
+    }
+
     fn write_file(&self, path: &SystemPath, content: &str) -> Result<()> {
         self.memory_fs.write_file(path, content)
     }

crates/ruff_db/src/vendored.rs

@@ -4,12 +4,12 @@ use std::fmt::{self, Debug};
 use std::io::{self, Read, Write};
 use std::sync::{Arc, Mutex, MutexGuard};
 
-use crate::file_revision::FileRevision;
 use zip::result::ZipResult;
 use zip::write::FileOptions;
 use zip::{CompressionMethod, ZipArchive, ZipWriter, read::ZipFile};
 
 pub use self::path::{VendoredPath, VendoredPathBuf};
+use crate::file_revision::FileRevision;
 
 mod path;
 

crates/ty_server/Cargo.toml

@@ -19,6 +19,7 @@ ruff_text_size = { workspace = true }
 ty_ide = { workspace = true }
 ty_project = { workspace = true }
 ty_python_semantic = { workspace = true }
+ty_vendored = { workspace = true }
 
 anyhow = { workspace = true }
 crossbeam = { workspace = true }

crates/ty_server/src/system.rs

@@ -8,7 +8,7 @@ use ruff_db::files::{File, FilePath};
 use ruff_db::system::walk_directory::WalkDirectoryBuilder;
 use ruff_db::system::{
     CaseSensitivity, DirectoryEntry, FileType, GlobError, Metadata, OsSystem, PatternError, Result,
-    System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf,
+    System, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf, WritableSystem,
 };
 use ruff_notebook::{Notebook, NotebookError};
 use ty_python_semantic::Db;
@@ -17,13 +17,29 @@ use crate::DocumentQuery;
 use crate::document::DocumentKey;
 use crate::session::index::Index;
 
+/// Returns a [`Url`] for the given [`File`].
 pub(crate) fn file_to_url(db: &dyn Db, file: File) -> Option<Url> {
     match file.path(db) {
         FilePath::System(system) => Url::from_file_path(system.as_std_path()).ok(),
         FilePath::SystemVirtual(path) => Url::parse(path.as_str()).ok(),
-        // TODO: Not yet supported, consider an approach similar to Sorbet's custom paths
+        FilePath::Vendored(path) => {
-        // https://sorbet.org/docs/sorbet-uris
+            let writable = db.system().as_writable()?;
-        FilePath::Vendored(_) => None,
+
+            let system_path = SystemPathBuf::from(format!(
+                "vendored/typeshed/{}/{}",
+                // The vendored files are uniquely identified by the source commit.
+                ty_vendored::SOURCE_COMMIT,
+                path.as_str()
+            ));
+
+            // Extract the vendored file onto the system.
+            let system_path = writable
+                .get_or_cache(&system_path, &|| db.vendored().read_to_string(path))
+                .ok()
+                .flatten()?;
+
+            Url::from_file_path(system_path.as_std_path()).ok()
+        }
     }
 }
 
@@ -224,6 +240,10 @@ impl System for LSPSystem {
         self.os_system.user_config_directory()
     }
 
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        self.os_system.cache_dir()
+    }
+
     fn read_directory<'a>(
         &'a self,
         path: &SystemPath,
@@ -245,6 +265,10 @@ impl System for LSPSystem {
         self.os_system.glob(pattern)
     }
 
+    fn as_writable(&self) -> Option<&dyn WritableSystem> {
+        self.os_system.as_writable()
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }

crates/ty_test/src/db.rs

@@ -226,6 +226,10 @@ impl System for MdtestSystem {
         self.as_system().user_config_directory()
     }
 
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        self.as_system().cache_dir()
+    }
+
     fn read_directory<'a>(
         &'a self,
         path: &SystemPath,
@@ -253,6 +257,10 @@ impl System for MdtestSystem {
             .glob(self.normalize_path(SystemPath::new(pattern)).as_str())
     }
 
+    fn as_writable(&self) -> Option<&dyn WritableSystem> {
+        Some(self)
+    }
+
     fn as_any(&self) -> &dyn std::any::Any {
         self
     }
@@ -263,6 +271,10 @@ impl System for MdtestSystem {
 }
 
 impl WritableSystem for MdtestSystem {
+    fn create_new_file(&self, path: &SystemPath) -> ruff_db::system::Result<()> {
+        self.as_system().create_new_file(&self.normalize_path(path))
+    }
+
     fn write_file(&self, path: &SystemPath, content: &str) -> ruff_db::system::Result<()> {
         self.as_system()
             .write_file(&self.normalize_path(path), content)

crates/ty_vendored/Cargo.toml

@@ -12,6 +12,7 @@ license = { workspace = true }
 
 [dependencies]
 ruff_db = { workspace = true }
+static_assertions = { workspace = true }
 zip = { workspace = true }
 
 [build-dependencies]

crates/ty_vendored/src/lib.rs

@@ -1,6 +1,12 @@
 use ruff_db::vendored::VendoredFileSystem;
 use std::sync::LazyLock;
 
+/// The source commit of the vendored typeshed.
+pub const SOURCE_COMMIT: &str =
+    include_str!("../../../crates/ty_vendored/vendor/typeshed/source_commit.txt").trim_ascii_end();
+
+static_assertions::const_assert_eq!(SOURCE_COMMIT.len(), 40);
+
 // The file path here is hardcoded in this crate's `build.rs` script.
 // Luckily this crate will fail to build if this file isn't available at build time.
 static TYPESHED_ZIP_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/zipped_typeshed.zip"));

crates/ty_wasm/src/lib.rs

@@ -8,7 +8,7 @@ use ruff_db::source::{line_index, source_text};
 use ruff_db::system::walk_directory::WalkDirectoryBuilder;
 use ruff_db::system::{
     CaseSensitivity, DirectoryEntry, GlobError, MemoryFileSystem, Metadata, PatternError, System,
-    SystemPath, SystemPathBuf, SystemVirtualPath,
+    SystemPath, SystemPathBuf, SystemVirtualPath, WritableSystem,
 };
 use ruff_notebook::Notebook;
 use ruff_python_formatter::formatted_file;
@@ -695,6 +695,10 @@ impl System for WasmSystem {
         None
     }
 
+    fn cache_dir(&self) -> Option<SystemPathBuf> {
+        None
+    }
+
     fn read_directory<'a>(
         &'a self,
         path: &SystemPath,
@@ -715,6 +719,10 @@ impl System for WasmSystem {
         Ok(Box::new(self.fs.glob(pattern)?))
     }
 
+    fn as_writable(&self) -> Option<&dyn WritableSystem> {
+        None
+    }
+
     fn as_any(&self) -> &dyn Any {
         self
     }