mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 10:22:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

[red-knot] Add support for untitled files (#12492)

Browse source 
## Summary

This PR adds support for untitled files in the Red Knot project.

Refer to the [design
discussion](https://github.com/astral-sh/ruff/discussions/12336) for
more details.

### Changes
* The `parsed_module` always assumes that the `SystemVirtual` path is of
`PySourceType::Python`.
* For the module resolver, as suggested, I went ahead by adding a new
`SystemOrVendoredPath` enum and renamed `FilePathRef` to
`SystemOrVendoredPathRef` (happy to consider better names here).
* The `file_to_module` query would return if it's a
`FilePath::SystemVirtual` variant because a virtual file doesn't belong
to any module.
* The sync implementation for the system virtual path is basically the
same as that of system path except that it uses the
`virtual_path_metadata`. The reason for this is that the system
(language server) would provide the metadata on whether it still exists
or not and if it exists, the corresponding metadata.

For point (1), VS Code would use `Untitled-1` for Python files and
`Untitled-1.ipynb` for Jupyter Notebooks. We could use this distinction
to determine whether the source type is `Python` or `Ipynb`.

## Test Plan

Added test cases in #12526
This commit is contained in:
Dhruv Manilawala 2024-07-26 18:13:31 +05:30 • committed by GitHub
parent 71f7aa4971
commit 6f4db8675b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 534 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

89
crates/red_knot_module_resolver/src/path.rs
View file

@ -5,7 +5,7 @@ use std::sync::Arc;
use camino::{Utf8Path, Utf8PathBuf};
use ruff_db::files::{system_path_to_file, vendored_path_to_file, File, FilePath};
use ruff_db::files::{system_path_to_file, vendored_path_to_file, File};
use ruff_db::system::{System, SystemPath, SystemPathBuf};
use ruff_db::vendored::{VendoredPath, VendoredPathBuf};
@ -474,18 +474,21 @@ impl SearchPath {
matches!(&*self.0, SearchPathInner::SitePackages(_))
}
#[must_use]
pub(crate) fn relativize_path(&self, path: &FilePath) -> Option<ModulePath> {
let extension = path.extension();
fn is_valid_extension(&self, extension: &str) -> bool {
if self.is_standard_library() {
if extension.is_some_and(|extension| extension != "pyi") {
return None;
}
extension == "pyi"
} else {
if extension.is_some_and(|extension| !matches!(extension, "pyi" | "py")) {
return None;
}
matches!(extension, "pyi" | "py")
}
}
#[must_use]
pub(crate) fn relativize_system_path(&self, path: &SystemPath) -> Option<ModulePath> {
if path
.extension()
.is_some_and(|extension| !self.is_valid_extension(extension))
{
return None;
}
match &*self.0 {
@ -493,16 +496,36 @@ impl SearchPath {
| SearchPathInner::FirstParty(search_path)
| SearchPathInner::StandardLibraryCustom(search_path)
| SearchPathInner::SitePackages(search_path)
| SearchPathInner::Editable(search_path) => path
.as_system_path()
.and_then(|absolute_path| absolute_path.strip_prefix(search_path).ok())
.map(|relative_path| ModulePath {
search_path: self.clone(),
relative_path: relative_path.as_utf8_path().to_path_buf(),
}),
| SearchPathInner::Editable(search_path) => {
path.strip_prefix(search_path)
.ok()
.map(|relative_path| ModulePath {
search_path: self.clone(),
relative_path: relative_path.as_utf8_path().to_path_buf(),
})
}
SearchPathInner::StandardLibraryVendored(_) => None,
}
}
#[must_use]
pub(crate) fn relativize_vendored_path(&self, path: &VendoredPath) -> Option<ModulePath> {
if path
.extension()
.is_some_and(|extension| !self.is_valid_extension(extension))
{
return None;
}
match &*self.0 {
SearchPathInner::Extra(_)
| SearchPathInner::FirstParty(_)
| SearchPathInner::StandardLibraryCustom(_)
| SearchPathInner::SitePackages(_)
| SearchPathInner::Editable(_) => None,
SearchPathInner::StandardLibraryVendored(search_path) => path
.as_vendored_path()
.and_then(|absolute_path| absolute_path.strip_prefix(search_path).ok())
.strip_prefix(search_path)
.ok()
.map(|relative_path| ModulePath {
search_path: self.clone(),
relative_path: relative_path.as_utf8_path().to_path_buf(),
@ -792,14 +815,14 @@ mod tests {
let root = SearchPath::custom_stdlib(&db, stdlib.parent().unwrap().to_path_buf()).unwrap();
// Must have a `.pyi` extension or no extension:
let bad_absolute_path = FilePath::system("foo/stdlib/x.py");
assert_eq!(root.relativize_path(&bad_absolute_path), None);
let second_bad_absolute_path = FilePath::system("foo/stdlib/x.rs");
assert_eq!(root.relativize_path(&second_bad_absolute_path), None);
let bad_absolute_path = SystemPath::new("foo/stdlib/x.py");
assert_eq!(root.relativize_system_path(bad_absolute_path), None);
let second_bad_absolute_path = SystemPath::new("foo/stdlib/x.rs");
assert_eq!(root.relativize_system_path(second_bad_absolute_path), None);
// Must be a path that is a child of `root`:
let third_bad_absolute_path = FilePath::system("bar/stdlib/x.pyi");
assert_eq!(root.relativize_path(&third_bad_absolute_path), None);
let third_bad_absolute_path = SystemPath::new("bar/stdlib/x.pyi");
assert_eq!(root.relativize_system_path(third_bad_absolute_path), None);
}
#[test]
@ -808,19 +831,21 @@ mod tests {
let root = SearchPath::extra(db.system(), src.clone()).unwrap();
// Must have a `.py` extension, a `.pyi` extension, or no extension:
let bad_absolute_path = FilePath::System(src.join("x.rs"));
assert_eq!(root.relativize_path(&bad_absolute_path), None);
let bad_absolute_path = src.join("x.rs");
assert_eq!(root.relativize_system_path(&bad_absolute_path), None);
// Must be a path that is a child of `root`:
let second_bad_absolute_path = FilePath::system("bar/src/x.pyi");
assert_eq!(root.relativize_path(&second_bad_absolute_path), None);
let second_bad_absolute_path = SystemPath::new("bar/src/x.pyi");
assert_eq!(root.relativize_system_path(second_bad_absolute_path), None);
}
#[test]
fn relativize_path() {
let TestCase { db, src, .. } = TestCaseBuilder::new().build();
let src_search_path = SearchPath::first_party(db.system(), src.clone()).unwrap();
let eggs_package = FilePath::System(src.join("eggs/__init__.pyi"));
let module_path = src_search_path.relativize_path(&eggs_package).unwrap();
let eggs_package = src.join("eggs/__init__.pyi");
let module_path = src_search_path
.relativize_system_path(&eggs_package)
.unwrap();
assert_eq!(
&module_path.relative_path,
Utf8Path::new("eggs/__init__.pyi")

19
crates/red_knot_module_resolver/src/resolver.rs
View file

@ -7,6 +7,7 @@ use rustc_hash::{FxBuildHasher, FxHashSet};
use ruff_db::files::{File, FilePath};
use ruff_db::program::{Program, SearchPathSettings, TargetVersion};
use ruff_db::system::{DirectoryEntry, System, SystemPath, SystemPathBuf};
use ruff_db::vendored::VendoredPath;
use crate::db::Db;
use crate::module::{Module, ModuleKind};
@ -57,6 +58,12 @@ pub(crate) fn path_to_module(db: &dyn Db, path: &FilePath) -> Option<Module> {
file_to_module(db, file)
}
#[derive(Debug, Clone, Copy)]
enum SystemOrVendoredPathRef<'a> {
System(&'a SystemPath),
Vendored(&'a VendoredPath),
}
/// Resolves the module for the file with the given id.
///
/// Returns `None` if the file is not a module locatable via any of the known search paths.
@ -64,7 +71,11 @@ pub(crate) fn path_to_module(db: &dyn Db, path: &FilePath) -> Option<Module> {
pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option<Module> {
let _span = tracing::trace_span!("file_to_module", ?file).entered();
let path = file.path(db.upcast());
let path = match file.path(db.upcast()) {
FilePath::System(system) => SystemOrVendoredPathRef::System(system),
FilePath::Vendored(vendored) => SystemOrVendoredPathRef::Vendored(vendored),
FilePath::SystemVirtual(_) => return None,
};
let settings = module_resolution_settings(db);
@ -72,7 +83,11 @@ pub(crate) fn file_to_module(db: &dyn Db, file: File) -> Option<Module> {
let module_name = loop {
let candidate = search_paths.next()?;
if let Some(relative_path) = candidate.relativize_path(path) {
let relative_path = match path {
SystemOrVendoredPathRef::System(path) => candidate.relativize_system_path(path),
SystemOrVendoredPathRef::Vendored(path) => candidate.relativize_vendored_path(path),
};
if let Some(relative_path) = relative_path {
break relative_path.to_module_name()?;
}
};

62
crates/ruff_db/src/files.rs
View file

@ -5,7 +5,7 @@ use dashmap::mapref::entry::Entry;
use crate::file_revision::FileRevision;
use crate::files::private::FileStatus;
use crate::system::{SystemPath, SystemPathBuf};
use crate::system::{Metadata, SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf};
use crate::vendored::{VendoredPath, VendoredPathBuf};
use crate::{Db, FxDashMap};
pub use path::FilePath;
@ -47,6 +47,9 @@ struct FilesInner {
/// so that queries that depend on the existence of a file are re-executed when the file is created.
system_by_path: FxDashMap<SystemPathBuf, File>,
/// Lookup table that maps [`SystemVirtualPathBuf`]s to salsa interned [`File`] instances.
system_virtual_by_path: FxDashMap<SystemVirtualPathBuf, File>,
/// Lookup table that maps vendored files to the salsa [`File`] ingredients.
vendored_by_path: FxDashMap<VendoredPathBuf, File>,
}
@ -126,6 +129,36 @@ impl Files {
Some(file)
}
/// Looks up a virtual file by its `path`.
///
/// For a non-existing file, creates a new salsa [`File`] ingredient and stores it for future lookups.
///
/// The operations fails if the system failed to provide a metadata for the path.
#[tracing::instrument(level = "trace", skip(self, db), ret)]
pub fn add_virtual_file(&self, db: &dyn Db, path: &SystemVirtualPath) -> Option<File> {
let file = match self.inner.system_virtual_by_path.entry(path.to_path_buf()) {
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let metadata = db.system().virtual_path_metadata(path).ok()?;
let file = File::new(
db,
FilePath::SystemVirtual(path.to_path_buf()),
metadata.permissions(),
metadata.revision(),
FileStatus::Exists,
Count::default(),
);
entry.insert(file);
file
}
};
Some(file)
}
/// Refreshes the state of all known files under `path` recursively.
///
/// The most common use case is to update the [`Files`] state after removing or moving a directory.
@ -227,6 +260,9 @@ impl File {
db.system().read_to_string(system)
}
FilePath::Vendored(vendored) => db.vendored().read_to_string(vendored),
FilePath::SystemVirtual(system_virtual) => {
db.system().read_virtual_path_to_string(system_virtual)
}
}
}
@ -248,6 +284,9 @@ impl File {
std::io::ErrorKind::InvalidInput,
"Reading a notebook from the vendored file system is not supported.",
))),
FilePath::SystemVirtual(system_virtual) => {
db.system().read_virtual_path_to_notebook(system_virtual)
}
}
}
@ -255,7 +294,7 @@ impl File {
#[tracing::instrument(level = "debug", skip(db))]
pub fn sync_path(db: &mut dyn Db, path: &SystemPath) {
let absolute = SystemPath::absolute(path, db.system().current_directory());
Self::sync_impl(db, &absolute, None);
Self::sync_system_path(db, &absolute, None);
}
/// Syncs the [`File`]'s state with the state of the file on the system.
@ -265,22 +304,33 @@ impl File {
match path {
FilePath::System(system) => {
Self::sync_impl(db, &system, Some(self));
Self::sync_system_path(db, &system, Some(self));
}
FilePath::Vendored(_) => {
// Readonly, can never be out of date.
}
FilePath::SystemVirtual(system_virtual) => {
Self::sync_system_virtual_path(db, &system_virtual, self);
}
}
}
/// Private method providing the implementation for [`Self::sync_path`] and [`Self::sync_path`].
fn sync_impl(db: &mut dyn Db, path: &SystemPath, file: Option<File>) {
fn sync_system_path(db: &mut dyn Db, path: &SystemPath, file: Option<File>) {
let Some(file) = file.or_else(|| db.files().try_system(db, path)) else {
return;
};
let metadata = db.system().path_metadata(path);
Self::sync_impl(db, metadata, file);
}
fn sync_system_virtual_path(db: &mut dyn Db, path: &SystemVirtualPath, file: File) {
let metadata = db.system().virtual_path_metadata(path);
Self::sync_impl(db, metadata, file);
}
/// Private method providing the implementation for [`Self::sync_system_path`] and
/// [`Self::sync_system_virtual_path`].
fn sync_impl(db: &mut dyn Db, metadata: crate::system::Result<Metadata>, file: File) {
let (status, revision, permission) = match metadata {
Ok(metadata) if metadata.file_type().is_file() => (
FileStatus::Exists,

37
crates/ruff_db/src/files/path.rs
View file

@ -1,5 +1,5 @@
use crate::files::{system_path_to_file, vendored_path_to_file, File};
use crate::system::{SystemPath, SystemPathBuf};
use crate::system::{SystemPath, SystemPathBuf, SystemVirtualPath, SystemVirtualPathBuf};
use crate::vendored::{VendoredPath, VendoredPathBuf};
use crate::Db;
@ -8,11 +8,14 @@ use crate::Db;
/// The path abstracts that files in Ruff can come from different sources:
///
/// * a file stored on the [host system](crate::system::System).
/// * a virtual file stored on the [host system](crate::system::System).
/// * a vendored file stored in the [vendored file system](crate::vendored::VendoredFileSystem).
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub enum FilePath {
/// Path to a file on the [host system](crate::system::System).
System(SystemPathBuf),
/// Path to a virtual file on the [host system](crate::system::System).
SystemVirtual(SystemVirtualPathBuf),
/// Path to a file vendored as part of Ruff. Stored in the [vendored file system](crate::vendored::VendoredFileSystem).
Vendored(VendoredPathBuf),
}
@ -30,7 +33,7 @@ impl FilePath {
pub fn into_system_path_buf(self) -> Option<SystemPathBuf> {
match self {
FilePath::System(path) => Some(path),
FilePath::Vendored(_) => None,
FilePath::Vendored(_) | FilePath::SystemVirtual(_) => None,
}
}
@ -39,7 +42,7 @@ impl FilePath {
pub fn as_system_path(&self) -> Option<&SystemPath> {
match self {
FilePath::System(path) => Some(path.as_path()),
FilePath::Vendored(_) => None,
FilePath::Vendored(_) | FilePath::SystemVirtual(_) => None,
}
}
@ -50,6 +53,14 @@ impl FilePath {
matches!(self, FilePath::System(_))
}
/// Returns `true` if the path is a file system path that is virtual i.e., it doesn't exists on
/// disk.
#[must_use]
#[inline]
pub const fn is_system_virtual_path(&self) -> bool {
matches!(self, FilePath::SystemVirtual(_))
}
/// Returns `true` if the path is a vendored path.
#[must_use]
#[inline]
@ -62,7 +73,7 @@ impl FilePath {
pub fn as_vendored_path(&self) -> Option<&VendoredPath> {
match self {
FilePath::Vendored(path) => Some(path.as_path()),
FilePath::System(_) => None,
FilePath::System(_) | FilePath::SystemVirtual(_) => None,
}
}
@ -71,6 +82,7 @@ impl FilePath {
match self {
FilePath::System(path) => path.as_str(),
FilePath::Vendored(path) => path.as_str(),
FilePath::SystemVirtual(path) => path.as_str(),
}
}
@ -78,12 +90,14 @@ impl FilePath {
///
/// Returns `Some` if a file for `path` exists and is accessible by the user. Returns `None` otherwise.
///
/// See [`system_path_to_file`] and [`vendored_path_to_file`] if you always have either a file system or vendored path.
/// See [`system_path_to_file`] or [`vendored_path_to_file`] if you always have either a file
/// system or vendored path.
#[inline]
pub fn to_file(&self, db: &dyn Db) -> Option<File> {
match self {
FilePath::System(path) => system_path_to_file(db, path),
FilePath::Vendored(path) => vendored_path_to_file(db, path),
FilePath::SystemVirtual(_) => None,
}
}
@ -92,6 +106,7 @@ impl FilePath {
match self {
FilePath::System(path) => path.extension(),
FilePath::Vendored(path) => path.extension(),
FilePath::SystemVirtual(_) => None,
}
}
}
@ -126,6 +141,18 @@ impl From<&VendoredPath> for FilePath {
}
}
impl From<&SystemVirtualPath> for FilePath {
fn from(value: &SystemVirtualPath) -> Self {
FilePath::SystemVirtual(value.to_path_buf())
}
}
impl From<SystemVirtualPathBuf> for FilePath {
fn from(value: SystemVirtualPathBuf) -> Self {
FilePath::SystemVirtual(value)
}
}
impl PartialEq<SystemPath> for FilePath {
#[inline]
fn eq(&self, other: &SystemPath) -> bool {

38
crates/ruff_db/src/parsed.rs
View file

@ -32,6 +32,9 @@ pub fn parsed_module(db: &dyn Db, file: File) -> ParsedModule {
.extension()
.map_or(PySourceType::Python, PySourceType::from_extension),
FilePath::Vendored(_) => PySourceType::Stub,
FilePath::SystemVirtual(path) => path
.extension()
.map_or(PySourceType::Python, PySourceType::from_extension),
};
ParsedModule::new(parse_unchecked_source(&source, ty))
@ -74,9 +77,10 @@ impl std::fmt::Debug for ParsedModule {
mod tests {
use crate::files::{system_path_to_file, vendored_path_to_file};
use crate::parsed::parsed_module;
use crate::system::{DbWithTestSystem, SystemPath};
use crate::system::{DbWithTestSystem, SystemPath, SystemVirtualPath};
use crate::tests::TestDb;
use crate::vendored::{tests::VendoredFileSystemBuilder, VendoredPath};
use crate::Db;
#[test]
fn python_file() -> crate::system::Result<()> {
@ -110,6 +114,38 @@ mod tests {
Ok(())
}
#[test]
fn virtual_python_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = SystemVirtualPath::new("untitled:Untitled-1");
db.write_virtual_file(path, "x = 10");
let file = db.files().add_virtual_file(&db, path).unwrap();
let parsed = parsed_module(&db, file);
assert!(parsed.is_valid());
Ok(())
}
#[test]
fn virtual_ipynb_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = SystemVirtualPath::new("untitled:Untitled-1.ipynb");
db.write_virtual_file(path, "%timeit a = b");
let file = db.files().add_virtual_file(&db, path).unwrap();
let parsed = parsed_module(&db, file);
assert!(parsed.is_valid());
Ok(())
}
#[test]
fn vendored_file() {
let mut db = TestDb::new();

40
crates/ruff_db/src/source.rs
View file

@ -8,7 +8,7 @@ use ruff_notebook::Notebook;
use ruff_python_ast::PySourceType;
use ruff_source_file::LineIndex;
use crate::files::File;
use crate::files::{File, FilePath};
use crate::Db;
/// Reads the source text of a python text file (must be valid UTF8) or notebook.
@ -16,25 +16,33 @@ use crate::Db;
pub fn source_text(db: &dyn Db, file: File) -> SourceText {
let _span = tracing::trace_span!("source_text", ?file).entered();
if let Some(path) = file.path(db).as_system_path() {
if path.extension().is_some_and(|extension| {
let is_notebook = match file.path(db) {
FilePath::System(system) => system.extension().is_some_and(|extension| {
PySourceType::try_from_extension(extension) == Some(PySourceType::Ipynb)
}) {
// TODO(micha): Proper error handling and emit a diagnostic. Tackle it together with `source_text`.
let notebook = file.read_to_notebook(db).unwrap_or_else(|error| {
tracing::error!("Failed to load notebook: {error}");
Notebook::empty()
});
return SourceText {
inner: Arc::new(SourceTextInner {
kind: SourceTextKind::Notebook(notebook),
count: Count::new(),
}),
};
}),
FilePath::SystemVirtual(system_virtual) => {
system_virtual.extension().is_some_and(|extension| {
PySourceType::try_from_extension(extension) == Some(PySourceType::Ipynb)
})
}
FilePath::Vendored(_) => false,
};
if is_notebook {
// TODO(micha): Proper error handling and emit a diagnostic. Tackle it together with `source_text`.
let notebook = file.read_to_notebook(db).unwrap_or_else(|error| {
tracing::error!("Failed to load notebook: {error}");
Notebook::empty()
});
return SourceText {
inner: Arc::new(SourceTextInner {
kind: SourceTextKind::Notebook(notebook),
count: Count::new(),
}),
};
}
let content = file.read_to_string(db).unwrap_or_else(|error| {
tracing::error!("Failed to load file: {error}");
String::default()

13
crates/ruff_db/src/system.rs
View file

@ -11,6 +11,7 @@ use crate::file_revision::FileRevision;
pub use self::path::{
deduplicate_nested_paths, DeduplicatedNestedPathsIter, SystemPath, SystemPathBuf,
SystemVirtualPath, SystemVirtualPathBuf,
};
mod memory_fs;
@ -50,6 +51,18 @@ pub trait System: Debug {
/// representation fall-back to deserializing the notebook from a string.
fn read_to_notebook(&self, path: &SystemPath) -> std::result::Result<Notebook, NotebookError>;
/// Reads the metadata of the virtual file at `path`.
fn virtual_path_metadata(&self, path: &SystemVirtualPath) -> Result<Metadata>;
/// Reads the content of the virtual file at `path` into a [`String`].
fn read_virtual_path_to_string(&self, path: &SystemVirtualPath) -> Result<String>;
/// Reads the content of the virtual file at `path` as a [`Notebook`].
fn read_virtual_path_to_notebook(
&self,
path: &SystemVirtualPath,
) -> std::result::Result<Notebook, NotebookError>;
/// Returns `true` if `path` exists.
fn path_exists(&self, path: &SystemPath) -> bool {
self.path_metadata(path).is_ok()

112
crates/ruff_db/src/system/memory_fs.rs
View file

@ -4,9 +4,13 @@ use std::sync::{Arc, RwLock, RwLockWriteGuard};
use camino::{Utf8Path, Utf8PathBuf};
use filetime::FileTime;
use rustc_hash::FxHashMap;
use ruff_notebook::{Notebook, NotebookError};
use crate::system::{
walk_directory, DirectoryEntry, FileType, Metadata, Result, SystemPath, SystemPathBuf,
SystemVirtualPath, SystemVirtualPathBuf,
};
use super::walk_directory::{
@ -50,6 +54,7 @@ impl MemoryFileSystem {
let fs = Self {
inner: Arc::new(MemoryFileSystemInner {
by_path: RwLock::new(BTreeMap::default()),
virtual_files: RwLock::new(FxHashMap::default()),
cwd: cwd.clone(),
}),
};
@ -134,6 +139,42 @@ impl MemoryFileSystem {
ruff_notebook::Notebook::from_source_code(&content)
}
pub(crate) fn virtual_path_metadata(
&self,
path: impl AsRef<SystemVirtualPath>,
) -> Result<Metadata> {
let virtual_files = self.inner.virtual_files.read().unwrap();
let file = virtual_files
.get(&path.as_ref().to_path_buf())
.ok_or_else(not_found)?;
Ok(Metadata {
revision: file.last_modified.into(),
permissions: Some(MemoryFileSystem::PERMISSION),
file_type: FileType::File,
})
}
pub(crate) fn read_virtual_path_to_string(
&self,
path: impl AsRef<SystemVirtualPath>,
) -> Result<String> {
let virtual_files = self.inner.virtual_files.read().unwrap();
let file = virtual_files
.get(&path.as_ref().to_path_buf())
.ok_or_else(not_found)?;
Ok(file.content.clone())
}
pub(crate) fn read_virtual_path_to_notebook(
&self,
path: &SystemVirtualPath,
) -> std::result::Result<Notebook, NotebookError> {
let content = self.read_virtual_path_to_string(path)?;
ruff_notebook::Notebook::from_source_code(&content)
}
pub fn exists(&self, path: &SystemPath) -> bool {
let by_path = self.inner.by_path.read().unwrap();
let normalized = self.normalize_path(path);
@ -141,6 +182,11 @@ impl MemoryFileSystem {
by_path.contains_key(&normalized)
}
pub fn virtual_path_exists(&self, path: &SystemVirtualPath) -> bool {
let virtual_files = self.inner.virtual_files.read().unwrap();
virtual_files.contains_key(&path.to_path_buf())
}
/// Writes the files to the file system.
///
/// The operation overrides existing files with the same normalized path.
@ -173,6 +219,26 @@ impl MemoryFileSystem {
Ok(())
}
/// Stores a new virtual file in the file system.
///
/// The operation overrides the content for an existing virtual file with the same `path`.
pub fn write_virtual_file(&self, path: impl AsRef<SystemVirtualPath>, content: impl ToString) {
let path = path.as_ref();
let mut virtual_files = self.inner.virtual_files.write().unwrap();
match virtual_files.entry(path.to_path_buf()) {
std::collections::hash_map::Entry::Vacant(entry) => {
entry.insert(File {
content: content.to_string(),
last_modified: FileTime::now(),
});
}
std::collections::hash_map::Entry::Occupied(mut entry) => {
entry.get_mut().content = content.to_string();
}
}
}
/// Returns a builder for walking the directory tree of `path`.
///
/// The only files that are ignored when setting `WalkDirectoryBuilder::standard_filters`
@ -201,6 +267,17 @@ impl MemoryFileSystem {
remove_file(self, path.as_ref())
}
pub fn remove_virtual_file(&self, path: impl AsRef<SystemVirtualPath>) -> Result<()> {
let mut virtual_files = self.inner.virtual_files.write().unwrap();
match virtual_files.entry(path.as_ref().to_path_buf()) {
std::collections::hash_map::Entry::Occupied(entry) => {
entry.remove();
Ok(())
}
std::collections::hash_map::Entry::Vacant(_) => Err(not_found()),
}
}
/// Sets the last modified timestamp of the file stored at `path` to now.
///
/// Creates a new file if the file at `path` doesn't exist.
@ -309,6 +386,7 @@ impl std::fmt::Debug for MemoryFileSystem {
struct MemoryFileSystemInner {
by_path: RwLock<BTreeMap<Utf8PathBuf, Entry>>,
virtual_files: RwLock<FxHashMap<SystemVirtualPathBuf, File>>,
cwd: SystemPathBuf,
}
@ -586,6 +664,7 @@ mod tests {
use crate::system::walk_directory::WalkState;
use crate::system::{
DirectoryEntry, FileType, MemoryFileSystem, Result, SystemPath, SystemPathBuf,
SystemVirtualPath,
};
/// Creates a file system with the given files.
@ -724,6 +803,18 @@ mod tests {
Ok(())
}
#[test]
fn write_virtual_file() {
let fs = MemoryFileSystem::new();
fs.write_virtual_file("a", "content");
let error = fs.read_to_string("a").unwrap_err();
assert_eq!(error.kind(), ErrorKind::NotFound);
assert_eq!(fs.read_virtual_path_to_string("a").unwrap(), "content");
}
#[test]
fn read() -> Result<()> {
let fs = MemoryFileSystem::new();
@ -760,6 +851,15 @@ mod tests {
Ok(())
}
#[test]
fn read_fails_if_virtual_path_doesnt_exit() {
let fs = MemoryFileSystem::new();
let error = fs.read_virtual_path_to_string("a").unwrap_err();
assert_eq!(error.kind(), ErrorKind::NotFound);
}
#[test]
fn remove_file() -> Result<()> {
let fs = with_files(["a/a.py", "b.py"]);
@ -777,6 +877,18 @@ mod tests {
Ok(())
}
#[test]
fn remove_virtual_file() {
let fs = MemoryFileSystem::new();
fs.write_virtual_file("a", "content");
fs.write_virtual_file("b", "content");
fs.remove_virtual_file("a").unwrap();
assert!(!fs.virtual_path_exists(SystemVirtualPath::new("a")));
assert!(fs.virtual_path_exists(SystemVirtualPath::new("b")));
}
#[test]
fn remove_non_existing_file() {
let fs = with_files(["b.py"]);

20
crates/ruff_db/src/system/os.rs
View file

@ -7,6 +7,7 @@ use ruff_notebook::{Notebook, NotebookError};
use crate::system::{
DirectoryEntry, FileType, Metadata, Result, System, SystemPath, SystemPathBuf,
SystemVirtualPath,
};
use super::walk_directory::{
@ -76,6 +77,21 @@ impl System for OsSystem {
Notebook::from_path(path.as_std_path())
}
fn virtual_path_metadata(&self, _path: &SystemVirtualPath) -> Result<Metadata> {
Err(not_found())
}
fn read_virtual_path_to_string(&self, _path: &SystemVirtualPath) -> Result<String> {
Err(not_found())
}
fn read_virtual_path_to_notebook(
&self,
_path: &SystemVirtualPath,
) -> std::result::Result<Notebook, NotebookError> {
Err(NotebookError::from(not_found()))
}
fn path_exists(&self, path: &SystemPath) -> bool {
path.as_std_path().exists()
}
@ -275,6 +291,10 @@ impl From<WalkState> for ignore::WalkState {
}
}
fn not_found() -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::NotFound, "No such file or directory")
}
#[cfg(test)]
mod tests {
use tempfile::TempDir;

131
crates/ruff_db/src/system/path.rs
View file

@ -593,6 +593,137 @@ impl ruff_cache::CacheKey for SystemPathBuf {
}
}
/// A slice of a virtual path on [`System`](super::System) (akin to [`str`]).
#[repr(transparent)]
pub struct SystemVirtualPath(str);
impl SystemVirtualPath {
pub fn new(path: &str) -> &SystemVirtualPath {
// SAFETY: SystemVirtualPath is marked as #[repr(transparent)] so the conversion from a
// *const str to a *const SystemVirtualPath is valid.
unsafe { &*(path as *const str as *const SystemVirtualPath) }
}
/// Converts the path to an owned [`SystemVirtualPathBuf`].
pub fn to_path_buf(&self) -> SystemVirtualPathBuf {
SystemVirtualPathBuf(self.0.to_string())
}
/// Extracts the file extension, if possible.
///
/// # Examples
///
/// ```
/// use ruff_db::system::SystemVirtualPath;
///
/// assert_eq!(None, SystemVirtualPath::new("untitled:Untitled-1").extension());
/// assert_eq!("ipynb", SystemVirtualPath::new("untitled:Untitled-1.ipynb").extension().unwrap());
/// assert_eq!("ipynb", SystemVirtualPath::new("vscode-notebook-cell:Untitled-1.ipynb").extension().unwrap());
/// ```
///
/// See [`Path::extension`] for more details.
pub fn extension(&self) -> Option<&str> {
Path::new(&self.0).extension().and_then(|ext| ext.to_str())
}
/// Returns the path as a string slice.
#[inline]
pub fn as_str(&self) -> &str {
&self.0
}
}
/// An owned, virtual path on [`System`](`super::System`) (akin to [`String`]).
#[derive(Eq, PartialEq, Clone, Hash, PartialOrd, Ord)]
pub struct SystemVirtualPathBuf(String);
impl SystemVirtualPathBuf {
#[inline]
pub fn as_path(&self) -> &SystemVirtualPath {
SystemVirtualPath::new(&self.0)
}
}
impl From<String> for SystemVirtualPathBuf {
fn from(value: String) -> Self {
SystemVirtualPathBuf(value)
}
}
impl AsRef<SystemVirtualPath> for SystemVirtualPathBuf {
#[inline]
fn as_ref(&self) -> &SystemVirtualPath {
self.as_path()
}
}
impl AsRef<SystemVirtualPath> for SystemVirtualPath {
#[inline]
fn as_ref(&self) -> &SystemVirtualPath {
self
}
}
impl AsRef<SystemVirtualPath> for str {
#[inline]
fn as_ref(&self) -> &SystemVirtualPath {
SystemVirtualPath::new(self)
}
}
impl AsRef<SystemVirtualPath> for String {
#[inline]
fn as_ref(&self) -> &SystemVirtualPath {
SystemVirtualPath::new(self)
}
}
impl Deref for SystemVirtualPathBuf {
type Target = SystemVirtualPath;
fn deref(&self) -> &Self::Target {
self.as_path()
}
}
impl std::fmt::Debug for SystemVirtualPath {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::fmt::Display for SystemVirtualPath {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::fmt::Debug for SystemVirtualPathBuf {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::fmt::Display for SystemVirtualPathBuf {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
#[cfg(feature = "cache")]
impl ruff_cache::CacheKey for SystemVirtualPath {
fn cache_key(&self, hasher: &mut ruff_cache::CacheKeyHasher) {
self.as_str().cache_key(hasher);
}
}
#[cfg(feature = "cache")]
impl ruff_cache::CacheKey for SystemVirtualPathBuf {
fn cache_key(&self, hasher: &mut ruff_cache::CacheKeyHasher) {
self.as_path().cache_key(hasher);
}
}
/// Deduplicates identical paths and removes nested paths.
///
/// # Examples

36
crates/ruff_db/src/system/test.rs
View file

@ -2,7 +2,9 @@ use ruff_notebook::{Notebook, NotebookError};
use ruff_python_trivia::textwrap;
use crate::files::File;
use crate::system::{DirectoryEntry, MemoryFileSystem, Metadata, Result, System, SystemPath};
use crate::system::{
DirectoryEntry, MemoryFileSystem, Metadata, Result, System, SystemPath, SystemVirtualPath,
};
use crate::Db;
use std::any::Any;
use std::panic::RefUnwindSafe;
@ -71,6 +73,30 @@ impl System for TestSystem {
}
}
fn virtual_path_metadata(&self, path: &SystemVirtualPath) -> Result<Metadata> {
match &self.inner {
TestSystemInner::Stub(fs) => fs.virtual_path_metadata(path),
TestSystemInner::System(system) => system.virtual_path_metadata(path),
}
}
fn read_virtual_path_to_string(&self, path: &SystemVirtualPath) -> Result<String> {
match &self.inner {
TestSystemInner::Stub(fs) => fs.read_virtual_path_to_string(path),
TestSystemInner::System(system) => system.read_virtual_path_to_string(path),
}
}
fn read_virtual_path_to_notebook(
&self,
path: &SystemVirtualPath,
) -> std::result::Result<Notebook, NotebookError> {
match &self.inner {
TestSystemInner::Stub(fs) => fs.read_virtual_path_to_notebook(path),
TestSystemInner::System(system) => system.read_virtual_path_to_notebook(path),
}
}
fn path_exists(&self, path: &SystemPath) -> bool {
match &self.inner {
TestSystemInner::Stub(fs) => fs.exists(path),
@ -151,6 +177,14 @@ pub trait DbWithTestSystem: Db + Sized {
result
}
/// Writes the content of the given virtual file.
fn write_virtual_file(&mut self, path: impl AsRef<SystemVirtualPath>, content: impl ToString) {
let path = path.as_ref();
self.test_system()
.memory_file_system()
.write_virtual_file(path, content);
}
/// Writes auto-dedented text to a file.
fn write_dedented(&mut self, path: &str, content: &str) -> crate::system::Result<()> {
self.write_file(path, textwrap::dedent(content))?;