mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-03 18:28:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

[red-knot] Rename FileSystem to System (#12214)

Browse source
This commit is contained in:
Micha Reiser 2024-07-09 09:20:51 +02:00 • committed by GitHub
parent 16a63c88cf
commit ac04380f36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 1432 additions and 1291 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/ruff_db/Cargo.toml
View file

@ -27,4 +27,3 @@ zip = { workspace = true }
[dev-dependencies]
insta = { workspace = true }
once_cell = { workspace = true }

259
crates/ruff_db/src/vfs.rs → crates/ruff_db/src/files.rs
View file

@ -3,13 +3,12 @@ use std::sync::Arc;
use countme::Count;
use dashmap::mapref::entry::Entry;
pub use crate::vendored::{VendoredPath, VendoredPathBuf};
pub use path::VfsPath;
pub use path::FilePath;
use crate::file_revision::FileRevision;
use crate::file_system::FileSystemPath;
use crate::vendored::VendoredFileSystem;
use crate::vfs::private::FileStatus;
use crate::files::private::FileStatus;
use crate::system::SystemPath;
use crate::vendored::VendoredPath;
use crate::{Db, FxDashMap};
mod path;
@ -18,8 +17,8 @@ mod path;
///
/// Returns `None` if the path doesn't exist, isn't accessible, or if the path points to a directory.
#[inline]
pub fn system_path_to_file(db: &dyn Db, path: impl AsRef<FileSystemPath>) -> Option<VfsFile> {
let file = db.vfs().file_system(db, path.as_ref());
pub fn system_path_to_file(db: &dyn Db, path: impl AsRef<SystemPath>) -> Option<File> {
let file = db.files().system(db, path.as_ref());
// It's important that `vfs.file_system` creates a `VfsFile` even for files that don't exist or don't
// exist anymore so that Salsa can track that the caller of this function depends on the existence of
@ -33,98 +32,53 @@ pub fn system_path_to_file(db: &dyn Db, path: impl AsRef<FileSystemPath>) -> Opt
/// Interns a vendored file path. Returns `Some` if the vendored file for `path` exists and `None` otherwise.
#[inline]
pub fn vendored_path_to_file(db: &dyn Db, path: impl AsRef<VendoredPath>) -> Option<VfsFile> {
db.vfs().vendored(db, path.as_ref())
pub fn vendored_path_to_file(db: &dyn Db, path: impl AsRef<VendoredPath>) -> Option<File> {
db.files().vendored(db, path.as_ref())
}
/// Interns a virtual file system path and returns a salsa [`VfsFile`] ingredient.
///
/// Returns `Some` if a file for `path` exists and is accessible by the user. Returns `None` otherwise.
///
/// See [`system_path_to_file`] and [`vendored_path_to_file`] if you always have either a file system or vendored path.
#[inline]
pub fn vfs_path_to_file(db: &dyn Db, path: &VfsPath) -> Option<VfsFile> {
match path {
VfsPath::FileSystem(path) => system_path_to_file(db, path),
VfsPath::Vendored(path) => vendored_path_to_file(db, path),
}
}
/// Virtual file system that supports files from different sources.
///
/// The [`Vfs`] supports accessing files from:
///
/// * The file system
/// * Vendored files that are part of the distributed Ruff binary
///
/// ## Why do both the [`Vfs`] and [`FileSystem`](crate::FileSystem) trait exist?
///
/// It would have been an option to define [`FileSystem`](crate::FileSystem) in a way that all its operation accept
/// a [`VfsPath`]. This would have allowed to unify most of [`Vfs`] and [`FileSystem`](crate::FileSystem). The reason why they are
/// separate is that not all operations are supported for all [`VfsPath`]s:
///
/// * The only relevant operations for [`VendoredPath`]s are testing for existence and reading the content.
/// * The vendored file system is immutable and doesn't support writing nor does it require watching for changes.
/// * There's no requirement to walk the vendored typesystem.
///
/// The other reason is that most operations know if they are working with vendored or file system paths.
/// Requiring them to convert the path to an `VfsPath` to test if the file exist is cumbersome.
///
/// The main downside of the approach is that vendored files needs their own stubbing mechanism.
/// Lookup table that maps [file paths](`FilePath`) to salsa interned [`File`] instances.
#[derive(Default)]
pub struct Vfs {
inner: Arc<VfsInner>,
pub struct Files {
inner: Arc<FilesInner>,
}
#[derive(Default)]
struct VfsInner {
/// Lookup table that maps [`VfsPath`]s to salsa interned [`VfsFile`] instances.
struct FilesInner {
/// Lookup table that maps [`FilePath`]s to salsa interned [`File`] instances.
///
/// The map also stores entries for files that don't exist on the file system. This is necessary
/// so that queries that depend on the existence of a file are re-executed when the file is created.
///
files_by_path: FxDashMap<VfsPath, VfsFile>,
vendored: VendoredVfs,
files_by_path: FxDashMap<FilePath, File>,
}
impl Vfs {
/// Creates a new [`Vfs`] instance where the vendored files are stubbed out.
pub fn with_stubbed_vendored() -> Self {
Self {
inner: Arc::new(VfsInner {
vendored: VendoredVfs::Stubbed(FxDashMap::default()),
..VfsInner::default()
}),
}
}
/// Looks up a file by its path.
impl Files {
/// Looks up a file by its `path`.
///
/// For a non-existing file, creates a new salsa [`VfsFile`] ingredient and stores it for future lookups.
/// For a non-existing file, creates a new salsa [`File`] ingredient and stores it for future lookups.
///
/// The operation always succeeds even if the path doesn't exist on disk, isn't accessible or if the path points to a directory.
/// In these cases, a file with status [`FileStatus::Deleted`] is returned.
#[tracing::instrument(level = "debug", skip(self, db))]
fn file_system(&self, db: &dyn Db, path: &FileSystemPath) -> VfsFile {
fn system(&self, db: &dyn Db, path: &SystemPath) -> File {
*self
.inner
.files_by_path
.entry(VfsPath::FileSystem(path.to_path_buf()))
.entry(FilePath::System(path.to_path_buf()))
.or_insert_with(|| {
let metadata = db.file_system().metadata(path);
let metadata = db.system().path_metadata(path);
match metadata {
Ok(metadata) if metadata.file_type().is_file() => VfsFile::new(
Ok(metadata) if metadata.file_type().is_file() => File::new(
db,
VfsPath::FileSystem(path.to_path_buf()),
FilePath::System(path.to_path_buf()),
metadata.permissions(),
metadata.revision(),
FileStatus::Exists,
Count::default(),
),
_ => VfsFile::new(
_ => File::new(
db,
VfsPath::FileSystem(path.to_path_buf()),
FilePath::System(path.to_path_buf()),
None,
FileRevision::zero(),
FileStatus::Deleted,
@ -134,24 +88,32 @@ impl Vfs {
})
}
/// Tries to look up the file for the given system path, returns `None` if no such file exists yet
fn try_system(&self, path: &SystemPath) -> Option<File> {
self.inner
.files_by_path
.get(&FilePath::System(path.to_path_buf()))
.map(|entry| *entry.value())
}
/// Looks up a vendored file by its path. Returns `Some` if a vendored file for the given path
/// exists and `None` otherwise.
#[tracing::instrument(level = "debug", skip(self, db))]
fn vendored(&self, db: &dyn Db, path: &VendoredPath) -> Option<VfsFile> {
fn vendored(&self, db: &dyn Db, path: &VendoredPath) -> Option<File> {
let file = match self
.inner
.files_by_path
.entry(VfsPath::Vendored(path.to_path_buf()))
.entry(FilePath::Vendored(path.to_path_buf()))
{
Entry::Occupied(entry) => *entry.get(),
Entry::Vacant(entry) => {
let revision = self.inner.vendored.revision(path)?;
let metadata = db.vendored().metadata(path).ok()?;
let file = VfsFile::new(
let file = File::new(
db,
VfsPath::Vendored(path.to_path_buf()),
FilePath::Vendored(path.to_path_buf()),
Some(0o444),
revision,
metadata.revision(),
FileStatus::Exists,
Count::default(),
);
@ -165,49 +127,16 @@ impl Vfs {
Some(file)
}
/// Stubs out the vendored files with the given content.
///
/// ## Panics
/// If there are pending snapshots referencing this `Vfs` instance.
pub fn stub_vendored<P, S>(&mut self, vendored: impl IntoIterator<Item = (P, S)>)
where
P: AsRef<VendoredPath>,
S: ToString,
{
let inner = Arc::get_mut(&mut self.inner).unwrap();
let stubbed = FxDashMap::default();
for (path, content) in vendored {
stubbed.insert(path.as_ref().to_path_buf(), content.to_string());
}
inner.vendored = VendoredVfs::Stubbed(stubbed);
}
/// Creates a salsa like snapshot of the files. The instances share
/// Creates a salsa like snapshot. The instances share
/// the same path-to-file mapping.
pub fn snapshot(&self) -> Self {
Self {
inner: self.inner.clone(),
}
}
fn read(&self, db: &dyn Db, path: &VfsPath) -> String {
match path {
VfsPath::FileSystem(path) => db.file_system().read(path).unwrap_or_default(),
VfsPath::Vendored(vendored) => db
.vfs()
.inner
.vendored
.read(vendored)
.expect("Vendored file to exist"),
}
}
}
impl std::fmt::Debug for Vfs {
impl std::fmt::Debug for Files {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut map = f.debug_map();
@ -218,12 +147,13 @@ impl std::fmt::Debug for Vfs {
}
}
/// A file that's either stored on the host system's file system or in the vendored file system.
#[salsa::input]
pub struct VfsFile {
pub struct File {
/// The path of the file.
#[id]
#[return_ref]
pub path: VfsPath,
pub path: FilePath,
/// The unix permissions of the file. Only supported on unix systems. Always `None` on Windows
/// or when the file has been deleted.
@ -234,17 +164,17 @@ pub struct VfsFile {
/// The status of the file.
///
/// Salsa doesn't support deleting inputs. The only way to signal to the depending queries that
/// Salsa doesn't support deleting inputs. The only way to signal dependent queries that
/// the file has been deleted is to change the status to `Deleted`.
status: FileStatus,
/// Counter that counts the number of created file instances and active file instances.
/// Only enabled in debug builds.
#[allow(unused)]
count: Count<VfsFile>,
count: Count<File>,
}
impl VfsFile {
impl File {
/// Reads the content of the file into a [`String`].
///
/// Reading the same file multiple times isn't guaranteed to return the same content. It's possible
@ -253,21 +183,26 @@ impl VfsFile {
/// an empty string, which is the closest to the content that the file contains now. Returning
/// an empty string shouldn't be a problem because the query will be re-executed as soon as the
/// changes are applied to the database.
pub(crate) fn read(&self, db: &dyn Db) -> String {
pub(crate) fn read_to_string(&self, db: &dyn Db) -> String {
let path = self.path(db);
if path.is_file_system_path() {
// Add a dependency on the revision to ensure the operation gets re-executed when the file changes.
let _ = self.revision(db);
}
let result = match path {
FilePath::System(system) => {
// Add a dependency on the revision to ensure the operation gets re-executed when the file changes.
let _ = self.revision(db);
db.vfs().read(db, path)
db.system().read_to_string(system)
}
FilePath::Vendored(vendored) => db.vendored().read_to_string(vendored),
};
result.unwrap_or_default()
}
/// Refreshes the file metadata by querying the file system if needed.
/// TODO: The API should instead take all observed changes from the file system directly
/// and then apply the VfsFile status accordingly. But for now, this is sufficient.
pub fn touch_path(db: &mut dyn Db, path: &VfsPath) {
pub fn touch_path(db: &mut dyn Db, path: &FilePath) {
Self::touch_impl(db, path, None);
}
@ -277,10 +212,10 @@ impl VfsFile {
}
/// Private method providing the implementation for [`Self::touch_path`] and [`Self::touch`].
fn touch_impl(db: &mut dyn Db, path: &VfsPath, file: Option<VfsFile>) {
fn touch_impl(db: &mut dyn Db, path: &FilePath, file: Option<File>) {
match path {
VfsPath::FileSystem(path) => {
let metadata = db.file_system().metadata(path);
FilePath::System(path) => {
let metadata = db.system().path_metadata(path);
let (status, revision) = match metadata {
Ok(metadata) if metadata.file_type().is_file() => {
@ -289,59 +224,20 @@ impl VfsFile {
_ => (FileStatus::Deleted, FileRevision::zero()),
};
let file = file.unwrap_or_else(|| db.vfs().file_system(db, path));
let Some(file) = file.or_else(|| db.files().try_system(path)) else {
return;
};
file.set_status(db).to(status);
file.set_revision(db).to(revision);
}
VfsPath::Vendored(_) => {
FilePath::Vendored(_) => {
// Readonly, can never be out of date.
}
}
}
}
#[derive(Debug)]
enum VendoredVfs {
#[allow(unused)]
Real(VendoredFileSystem),
Stubbed(FxDashMap<VendoredPathBuf, String>),
}
impl Default for VendoredVfs {
fn default() -> Self {
Self::Stubbed(FxDashMap::default())
}
}
impl VendoredVfs {
fn revision(&self, path: &VendoredPath) -> Option<FileRevision> {
match self {
VendoredVfs::Real(file_system) => file_system
.metadata(path)
.map(|metadata| metadata.revision()),
VendoredVfs::Stubbed(stubbed) => stubbed
.contains_key(&path.to_path_buf())
.then_some(FileRevision::new(1)),
}
}
fn read(&self, path: &VendoredPath) -> std::io::Result<String> {
match self {
VendoredVfs::Real(file_system) => file_system.read(path),
VendoredVfs::Stubbed(stubbed) => {
if let Some(contents) = stubbed.get(&path.to_path_buf()).as_deref().cloned() {
Ok(contents)
} else {
Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("Could not find file {path:?}"),
))
}
}
}
}
}
// The types in here need to be public because they're salsa ingredients but we
// don't want them to be publicly accessible. That's why we put them into a private module.
mod private {
@ -358,21 +254,22 @@ mod private {
#[cfg(test)]
mod tests {
use crate::file_revision::FileRevision;
use crate::files::{system_path_to_file, vendored_path_to_file};
use crate::system::DbWithTestSystem;
use crate::tests::TestDb;
use crate::vfs::{system_path_to_file, vendored_path_to_file};
use crate::vendored::tests::VendoredFileSystemBuilder;
#[test]
fn file_system_existing_file() -> crate::file_system::Result<()> {
fn file_system_existing_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
db.file_system_mut()
.write_files([("test.py", "print('Hello world')")])?;
db.write_file("test.py", "print('Hello world')")?;
let test = system_path_to_file(&db, "test.py").expect("File to exist.");
assert_eq!(test.permissions(&db), Some(0o755));
assert_ne!(test.revision(&db), FileRevision::zero());
assert_eq!(&test.read(&db), "print('Hello world')");
assert_eq!(&test.read_to_string(&db), "print('Hello world')");
Ok(())
}
@ -390,14 +287,18 @@ mod tests {
fn stubbed_vendored_file() {
let mut db = TestDb::new();
db.vfs_mut()
.stub_vendored([("test.py", "def foo() -> str")]);
let mut vendored_builder = VendoredFileSystemBuilder::new();
vendored_builder
.add_file("test.pyi", "def foo() -> str")
.unwrap();
let vendored = vendored_builder.finish().unwrap();
db.with_vendored(vendored);
let test = vendored_path_to_file(&db, "test.py").expect("Vendored file to exist.");
let test = vendored_path_to_file(&db, "test.pyi").expect("Vendored file to exist.");
assert_eq!(test.permissions(&db), Some(0o444));
assert_ne!(test.revision(&db), FileRevision::zero());
assert_eq!(&test.read(&db), "def foo() -> str");
assert_eq!(&test.read_to_string(&db), "def foo() -> str");
}
#[test]

176
crates/ruff_db/src/files/path.rs Normal file
View file

@ -0,0 +1,176 @@
use crate::files::{system_path_to_file, vendored_path_to_file, File};
use crate::system::{SystemPath, SystemPathBuf};
use crate::vendored::{VendoredPath, VendoredPathBuf};
use crate::Db;
/// Path to a file.
///
/// The path abstracts that files in Ruff can come from different sources:
///
/// * a file stored on the [host system](crate::system::System).
/// * a vendored file stored in the [vendored file system](crate::vendored::VendoredFileSystem).
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub enum FilePath {
/// Path to a file on the [host system](crate::system::System).
System(SystemPathBuf),
/// Path to a file vendored as part of Ruff. Stored in the [vendored file system](crate::vendored::VendoredFileSystem).
Vendored(VendoredPathBuf),
}
impl FilePath {
/// Create a new path to a file on the file system.
#[must_use]
pub fn system(path: impl AsRef<SystemPath>) -> Self {
FilePath::System(path.as_ref().to_path_buf())
}
/// Returns `Some` if the path is a file system path that points to a path on disk.
#[must_use]
#[inline]
pub fn into_system_path_buf(self) -> Option<SystemPathBuf> {
match self {
FilePath::System(path) => Some(path),
FilePath::Vendored(_) => None,
}
}
#[must_use]
#[inline]
pub fn as_system_path(&self) -> Option<&SystemPath> {
match self {
FilePath::System(path) => Some(path.as_path()),
FilePath::Vendored(_) => None,
}
}
/// Returns `true` if the path is a file system path that points to a path on disk.
#[must_use]
#[inline]
pub const fn is_system_path(&self) -> bool {
matches!(self, FilePath::System(_))
}
/// Returns `true` if the path is a vendored path.
#[must_use]
#[inline]
pub const fn is_vendored_path(&self) -> bool {
matches!(self, FilePath::Vendored(_))
}
#[must_use]
#[inline]
pub fn as_vendored_path(&self) -> Option<&VendoredPath> {
match self {
FilePath::Vendored(path) => Some(path.as_path()),
FilePath::System(_) => None,
}
}
/// Yields the underlying [`str`] slice.
pub fn as_str(&self) -> &str {
match self {
FilePath::System(path) => path.as_str(),
FilePath::Vendored(path) => path.as_str(),
}
}
/// Interns a virtual file system path and returns a salsa [`File`] ingredient.
///
/// Returns `Some` if a file for `path` exists and is accessible by the user. Returns `None` otherwise.
///
/// See [`system_path_to_file`] and [`vendored_path_to_file`] if you always have either a file system or vendored path.
#[inline]
pub fn to_file(&self, db: &dyn Db) -> Option<File> {
match self {
FilePath::System(path) => system_path_to_file(db, path),
FilePath::Vendored(path) => vendored_path_to_file(db, path),
}
}
}
impl AsRef<str> for FilePath {
fn as_ref(&self) -> &str {
self.as_str()
}
}
impl From<SystemPathBuf> for FilePath {
fn from(value: SystemPathBuf) -> Self {
Self::System(value)
}
}
impl From<&SystemPath> for FilePath {
fn from(value: &SystemPath) -> Self {
FilePath::System(value.to_path_buf())
}
}
impl From<VendoredPathBuf> for FilePath {
fn from(value: VendoredPathBuf) -> Self {
Self::Vendored(value)
}
}
impl From<&VendoredPath> for FilePath {
fn from(value: &VendoredPath) -> Self {
Self::Vendored(value.to_path_buf())
}
}
impl PartialEq<SystemPath> for FilePath {
#[inline]
fn eq(&self, other: &SystemPath) -> bool {
self.as_system_path()
.is_some_and(|self_path| self_path == other)
}
}
impl PartialEq<FilePath> for SystemPath {
#[inline]
fn eq(&self, other: &FilePath) -> bool {
other == self
}
}
impl PartialEq<SystemPathBuf> for FilePath {
#[inline]
fn eq(&self, other: &SystemPathBuf) -> bool {
self == other.as_path()
}
}
impl PartialEq<FilePath> for SystemPathBuf {
fn eq(&self, other: &FilePath) -> bool {
other == self
}
}
impl PartialEq<VendoredPath> for FilePath {
#[inline]
fn eq(&self, other: &VendoredPath) -> bool {
self.as_vendored_path()
.is_some_and(|self_path| self_path == other)
}
}
impl PartialEq<FilePath> for VendoredPath {
#[inline]
fn eq(&self, other: &FilePath) -> bool {
other == self
}
}
impl PartialEq<VendoredPathBuf> for FilePath {
#[inline]
fn eq(&self, other: &VendoredPathBuf) -> bool {
other.as_path() == self
}
}
impl PartialEq<FilePath> for VendoredPathBuf {
#[inline]
fn eq(&self, other: &FilePath) -> bool {
other == self
}
}

79
crates/ruff_db/src/lib.rs
View file

@ -3,28 +3,29 @@ use std::hash::BuildHasherDefault;
use rustc_hash::FxHasher;
use salsa::DbWithJar;
use crate::file_system::FileSystem;
use crate::files::{File, Files};
use crate::parsed::parsed_module;
use crate::source::{line_index, source_text};
use crate::vfs::{Vfs, VfsFile};
use crate::system::System;
use crate::vendored::VendoredFileSystem;
pub mod file_revision;
pub mod file_system;
pub mod files;
pub mod parsed;
pub mod source;
pub mod system;
pub mod vendored;
pub mod vfs;
pub(crate) type FxDashMap<K, V> = dashmap::DashMap<K, V, BuildHasherDefault<FxHasher>>;
#[salsa::jar(db=Db)]
pub struct Jar(VfsFile, source_text, line_index, parsed_module);
pub struct Jar(File, source_text, line_index, parsed_module);
/// Database that gives access to the virtual filesystem, source code, and parsed AST.
/// Most basic database that gives access to files, the host system, source code, and parsed AST.
pub trait Db: DbWithJar<Jar> {
fn file_system(&self) -> &dyn FileSystem;
fn vfs(&self) -> &Vfs;
fn vendored(&self) -> &VendoredFileSystem;
fn system(&self) -> &dyn System;
fn files(&self) -> &Files;
}
/// Trait for upcasting a reference to a base trait object.
@ -38,39 +39,36 @@ mod tests {
use salsa::DebugWithDb;
use crate::file_system::{FileSystem, MemoryFileSystem};
use crate::vfs::{VendoredPathBuf, Vfs};
use crate::files::Files;
use crate::system::TestSystem;
use crate::system::{DbWithTestSystem, System};
use crate::vendored::VendoredFileSystem;
use crate::{Db, Jar};
/// Database that can be used for testing.
///
/// Uses an in memory filesystem and it stubs out the vendored files by default.
#[derive(Default)]
#[salsa::db(Jar)]
pub(crate) struct TestDb {
storage: salsa::Storage<Self>,
vfs: Vfs,
file_system: MemoryFileSystem,
files: Files,
system: TestSystem,
vendored: VendoredFileSystem,
events: std::sync::Arc<std::sync::Mutex<Vec<salsa::Event>>>,
}
impl TestDb {
pub(crate) fn new() -> Self {
let mut vfs = Vfs::default();
vfs.stub_vendored::<VendoredPathBuf, String>([]);
Self {
storage: salsa::Storage::default(),
file_system: MemoryFileSystem::default(),
system: TestSystem::default(),
vendored: VendoredFileSystem::default(),
events: std::sync::Arc::default(),
vfs,
files: Files::default(),
}
}
#[allow(unused)]
pub(crate) fn file_system(&self) -> &MemoryFileSystem {
&self.file_system
}
/// Empties the internal store of salsa events that have been emitted,
/// and returns them as a `Vec` (equivalent to [`std::mem::take`]).
///
@ -93,22 +91,32 @@ mod tests {
self.take_salsa_events();
}
pub(crate) fn file_system_mut(&mut self) -> &mut MemoryFileSystem {
&mut self.file_system
}
pub(crate) fn vfs_mut(&mut self) -> &mut Vfs {
&mut self.vfs
pub(crate) fn with_vendored(&mut self, vendored_file_system: VendoredFileSystem) {
self.vendored = vendored_file_system;
}
}
impl Db for TestDb {
fn file_system(&self) -> &dyn FileSystem {
&self.file_system
fn vendored(&self) -> &VendoredFileSystem {
&self.vendored
}
fn vfs(&self) -> &Vfs {
&self.vfs
fn system(&self) -> &dyn System {
&self.system
}
fn files(&self) -> &Files {
&self.files
}
}
impl DbWithTestSystem for TestDb {
fn test_system(&self) -> &TestSystem {
&self.system
}
fn test_system_mut(&mut self) -> &mut TestSystem {
&mut self.system
}
}
@ -124,9 +132,10 @@ mod tests {
fn snapshot(&self) -> salsa::Snapshot<Self> {
salsa::Snapshot::new(Self {
storage: self.storage.snapshot(),
file_system: self.file_system.snapshot(),
vfs: self.vfs.snapshot(),
system: self.system.snapshot(),
files: self.files.snapshot(),
events: self.events.clone(),
vendored: self.vendored.snapshot(),
})
}
}

42
crates/ruff_db/src/parsed.rs
View file

@ -5,13 +5,13 @@ use std::sync::Arc;
use ruff_python_ast::{ModModule, PySourceType};
use ruff_python_parser::{parse_unchecked_source, Parsed};
use crate::files::{File, FilePath};
use crate::source::source_text;
use crate::vfs::{VfsFile, VfsPath};
use crate::Db;
/// Returns the parsed AST of `file`, including its token stream.
///
/// The query uses Ruff's error-resilient parser. That means that the parser always succeeds to produce a
/// The query uses Ruff's error-resilient parser. That means that the parser always succeeds to produce an
/// AST even if the file contains syntax errors. The parse errors
/// are then accessible through [`Parsed::errors`].
///
@ -21,17 +21,17 @@ use crate::Db;
/// The other reason is that Ruff's AST doesn't implement `Eq` which Sala requires
/// for determining if a query result is unchanged.
#[salsa::tracked(return_ref, no_eq)]
pub fn parsed_module(db: &dyn Db, file: VfsFile) -> ParsedModule {
pub fn parsed_module(db: &dyn Db, file: File) -> ParsedModule {
let _span = tracing::trace_span!("parse_module", file = ?file).entered();
let source = source_text(db, file);
let path = file.path(db);
let ty = match path {
VfsPath::FileSystem(path) => path
FilePath::System(path) => path
.extension()
.map_or(PySourceType::Python, PySourceType::from_extension),
VfsPath::Vendored(_) => PySourceType::Stub,
FilePath::Vendored(_) => PySourceType::Stub,
};
ParsedModule::new(parse_unchecked_source(&source, ty))
@ -72,19 +72,18 @@ impl std::fmt::Debug for ParsedModule {
#[cfg(test)]
mod tests {
use crate::file_system::FileSystemPath;
use crate::files::{system_path_to_file, vendored_path_to_file};
use crate::parsed::parsed_module;
use crate::system::{DbWithTestSystem, SystemPath};
use crate::tests::TestDb;
use crate::vendored::VendoredPath;
use crate::vfs::{system_path_to_file, vendored_path_to_file};
use crate::vendored::{tests::VendoredFileSystemBuilder, VendoredPath};
#[test]
fn python_file() -> crate::file_system::Result<()> {
fn python_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = "test.py";
db.file_system_mut()
.write_file(path, "x = 10".to_string())?;
db.write_file(path, "x = 10".to_string())?;
let file = system_path_to_file(&db, path).unwrap();
@ -96,12 +95,11 @@ mod tests {
}
#[test]
fn python_ipynb_file() -> crate::file_system::Result<()> {
fn python_ipynb_file() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = FileSystemPath::new("test.ipynb");
let path = SystemPath::new("test.ipynb");
db.file_system_mut()
.write_file(path, "%timeit a = b".to_string())?;
db.write_file(path, "%timeit a = b".to_string())?;
let file = system_path_to_file(&db, path).unwrap();
@ -115,9 +113,12 @@ mod tests {
#[test]
fn vendored_file() {
let mut db = TestDb::new();
db.vfs_mut().stub_vendored([(
"path.pyi",
r#"
let mut vendored_builder = VendoredFileSystemBuilder::new();
vendored_builder
.add_file(
"path.pyi",
r#"
import sys
if sys.platform == "win32":
@ -126,7 +127,10 @@ if sys.platform == "win32":
else:
from posixpath import *
from posixpath import __all__ as __all__"#,
)]);
)
.unwrap();
let vendored = vendored_builder.finish().unwrap();
db.with_vendored(vendored);
let file = vendored_path_to_file(&db, VendoredPath::new("path.pyi")).unwrap();

45
crates/ruff_db/src/source.rs
View file

@ -4,15 +4,15 @@ use salsa::DebugWithDb;
use std::ops::Deref;
use std::sync::Arc;
use crate::vfs::VfsFile;
use crate::files::File;
use crate::Db;
/// Reads the content of file.
#[salsa::tracked]
pub fn source_text(db: &dyn Db, file: VfsFile) -> SourceText {
pub fn source_text(db: &dyn Db, file: File) -> SourceText {
let _span = tracing::trace_span!("source_text", ?file).entered();
let content = file.read(db);
let content = file.read_to_string(db);
SourceText {
inner: Arc::from(content),
@ -22,7 +22,7 @@ pub fn source_text(db: &dyn Db, file: VfsFile) -> SourceText {
/// Computes the [`LineIndex`] for `file`.
#[salsa::tracked]
pub fn line_index(db: &dyn Db, file: VfsFile) -> LineIndex {
pub fn line_index(db: &dyn Db, file: File) -> LineIndex {
let _span = tracing::trace_span!("line_index", file = ?file.debug(db)).entered();
let source = source_text(db, file);
@ -30,7 +30,7 @@ pub fn line_index(db: &dyn Db, file: VfsFile) -> LineIndex {
LineIndex::from_source_text(&source)
}
/// The source text of a [`VfsFile`].
/// The source text of a [`File`].
///
/// Cheap cloneable in `O(1)`.
#[derive(Clone, Eq, PartialEq)]
@ -63,30 +63,25 @@ impl std::fmt::Debug for SourceText {
mod tests {
use salsa::EventKind;
use crate::files::system_path_to_file;
use crate::source::{line_index, source_text};
use crate::system::{DbWithTestSystem, SystemPath};
use crate::tests::TestDb;
use ruff_source_file::OneIndexed;
use ruff_text_size::TextSize;
use crate::file_system::FileSystemPath;
use crate::source::{line_index, source_text};
use crate::tests::TestDb;
use crate::vfs::system_path_to_file;
#[test]
fn re_runs_query_when_file_revision_changes() -> crate::file_system::Result<()> {
fn re_runs_query_when_file_revision_changes() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = FileSystemPath::new("test.py");
let path = SystemPath::new("test.py");
db.file_system_mut()
.write_file(path, "x = 10".to_string())?;
db.write_file(path, "x = 10".to_string())?;
let file = system_path_to_file(&db, path).unwrap();
assert_eq!(&*source_text(&db, file), "x = 10");
db.file_system_mut()
.write_file(path, "x = 20".to_string())
.unwrap();
file.touch(&mut db);
db.write_file(path, "x = 20".to_string()).unwrap();
assert_eq!(&*source_text(&db, file), "x = 20");
@ -94,12 +89,11 @@ mod tests {
}
#[test]
fn text_is_cached_if_revision_is_unchanged() -> crate::file_system::Result<()> {
fn text_is_cached_if_revision_is_unchanged() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = FileSystemPath::new("test.py");
let path = SystemPath::new("test.py");
db.file_system_mut()
.write_file(path, "x = 10".to_string())?;
db.write_file(path, "x = 10".to_string())?;
let file = system_path_to_file(&db, path).unwrap();
@ -121,12 +115,11 @@ mod tests {
}
#[test]
fn line_index_for_source() -> crate::file_system::Result<()> {
fn line_index_for_source() -> crate::system::Result<()> {
let mut db = TestDb::new();
let path = FileSystemPath::new("test.py");
let path = SystemPath::new("test.py");
db.file_system_mut()
.write_file(path, "x = 10\ny = 20".to_string())?;
db.write_file(path, "x = 10\ny = 20".to_string())?;
let file = system_path_to_file(&db, path).unwrap();
let index = line_index(&db, file);

97
crates/ruff_db/src/system.rs Normal file
View file

@ -0,0 +1,97 @@
pub use memory_fs::MemoryFileSystem;
pub use os::OsSystem;
pub use test::{DbWithTestSystem, TestSystem};
use crate::file_revision::FileRevision;
pub use self::path::{SystemPath, SystemPathBuf};
mod memory_fs;
mod os;
mod path;
mod test;
pub type Result<T> = std::io::Result<T>;
/// The system on which Ruff runs.
///
/// Ruff supports running on the CLI, in a language server, and in a browser (WASM). Each of these
/// host-systems differ in what system operations they support and how they interact with the file system:
/// * Language server:
/// * Reading a file's content should take into account that it might have unsaved changes because it's open in the editor.
/// * Use structured representations for notebooks, making deserializing a notebook from a string unnecessary.
/// * Use their own file watching infrastructure.
/// * WASM (Browser):
/// * There are ways to emulate a file system in WASM but a native memory-filesystem is more efficient.
/// * Doesn't support a current working directory
/// * File watching isn't supported.
///
/// Abstracting the system also enables tests to use a more efficient in-memory file system.
pub trait System {
/// Reads the metadata of the file or directory at `path`.
fn path_metadata(&self, path: &SystemPath) -> Result<Metadata>;
/// Reads the content of the file at `path` into a [`String`].
fn read_to_string(&self, path: &SystemPath) -> Result<String>;
/// Returns `true` if `path` exists.
fn path_exists(&self, path: &SystemPath) -> bool {
self.path_metadata(path).is_ok()
}
/// Returns `true` if `path` exists and is a directory.
fn is_directory(&self, path: &SystemPath) -> bool {
self.path_metadata(path)
.map_or(false, |metadata| metadata.file_type.is_directory())
}
/// Returns `true` if `path` exists and is a file.
fn is_file(&self, path: &SystemPath) -> bool {
self.path_metadata(path)
.map_or(false, |metadata| metadata.file_type.is_file())
}
fn as_any(&self) -> &dyn std::any::Any;
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Metadata {
revision: FileRevision,
permissions: Option<u32>,
file_type: FileType,
}
impl Metadata {
pub fn revision(&self) -> FileRevision {
self.revision
}
pub fn permissions(&self) -> Option<u32> {
self.permissions
}
pub fn file_type(&self) -> FileType {
self.file_type
}
}
#[derive(Copy, Clone, Eq, PartialEq, Debug, Hash)]
pub enum FileType {
File,
Directory,
Symlink,
}
impl FileType {
pub const fn is_file(self) -> bool {
matches!(self, FileType::File)
}
pub const fn is_directory(self) -> bool {
matches!(self, FileType::Directory)
}
pub const fn is_symlink(self) -> bool {
matches!(self, FileType::Symlink)
}
}

271
crates/ruff_db/src/file_system/memory.rs → crates/ruff_db/src/system/memory_fs.rs
View file

@ -4,7 +4,7 @@ use std::sync::{Arc, RwLock, RwLockWriteGuard};
use camino::{Utf8Path, Utf8PathBuf};
use filetime::FileTime;
use crate::file_system::{FileSystem, FileSystemPath, FileType, Metadata, Result};
use crate::system::{FileType, Metadata, Result, SystemPath};
/// File system that stores all content in memory.
///
@ -16,9 +16,7 @@ use crate::file_system::{FileSystem, FileSystemPath, FileType, Metadata, Result}
/// * hardlinks
/// * permissions: All files and directories have the permission 0755.
///
/// Use a tempdir with the real file system to test these advanced file system features and complex file system behavior.
///
/// Only intended for testing purposes.
/// Use a tempdir with the real file system to test these advanced file system features and behavior.
#[derive(Clone)]
pub struct MemoryFileSystem {
inner: Arc<MemoryFileSystemInner>,
@ -32,7 +30,8 @@ impl MemoryFileSystem {
Self::with_cwd("/")
}
pub fn with_cwd(cwd: impl AsRef<FileSystemPath>) -> Self {
/// Creates a new, empty in memory file system with the given current working directory.
pub fn with_cwd(cwd: impl AsRef<SystemPath>) -> Self {
let cwd = Utf8PathBuf::from(cwd.as_ref().as_str());
assert!(
@ -47,7 +46,7 @@ impl MemoryFileSystem {
}),
};
fs.create_directory_all(FileSystemPath::new(&cwd)).unwrap();
fs.create_directory_all(SystemPath::new(&cwd)).unwrap();
fs
}
@ -59,6 +58,69 @@ impl MemoryFileSystem {
}
}
pub fn metadata(&self, path: impl AsRef<SystemPath>) -> Result<Metadata> {
fn metadata(fs: &MemoryFileSystemInner, path: &SystemPath) -> Result<Metadata> {
let by_path = fs.by_path.read().unwrap();
let normalized = normalize_path(path, &fs.cwd);
let entry = by_path.get(&normalized).ok_or_else(not_found)?;
let metadata = match entry {
Entry::File(file) => Metadata {
revision: file.last_modified.into(),
permissions: Some(MemoryFileSystem::PERMISSION),
file_type: FileType::File,
},
Entry::Directory(directory) => Metadata {
revision: directory.last_modified.into(),
permissions: Some(MemoryFileSystem::PERMISSION),
file_type: FileType::Directory,
},
};
Ok(metadata)
}
metadata(&self.inner, path.as_ref())
}
pub fn is_file(&self, path: impl AsRef<SystemPath>) -> bool {
let by_path = self.inner.by_path.read().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
matches!(by_path.get(&normalized), Some(Entry::File(_)))
}
pub fn is_directory(&self, path: impl AsRef<SystemPath>) -> bool {
let by_path = self.inner.by_path.read().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
matches!(by_path.get(&normalized), Some(Entry::Directory(_)))
}
pub fn read_to_string(&self, path: impl AsRef<SystemPath>) -> Result<String> {
fn read_to_string(fs: &MemoryFileSystemInner, path: &SystemPath) -> Result<String> {
let by_path = fs.by_path.read().unwrap();
let normalized = normalize_path(path, &fs.cwd);
let entry = by_path.get(&normalized).ok_or_else(not_found)?;
match entry {
Entry::File(file) => Ok(file.content.clone()),
Entry::Directory(_) => Err(is_a_directory()),
}
}
read_to_string(&self.inner, path.as_ref())
}
pub fn exists(&self, path: &SystemPath) -> bool {
let by_path = self.inner.by_path.read().unwrap();
let normalized = normalize_path(path, &self.inner.cwd);
by_path.contains_key(&normalized)
}
/// Writes the files to the file system.
///
/// The operation overrides existing files with the same normalized path.
@ -66,7 +128,7 @@ impl MemoryFileSystem {
/// Enclosing directories are automatically created if they don't exist.
pub fn write_files<P, C>(&self, files: impl IntoIterator<Item = (P, C)>) -> Result<()>
where
P: AsRef<FileSystemPath>,
P: AsRef<SystemPath>,
C: ToString,
{
for (path, content) in files {
@ -81,11 +143,7 @@ impl MemoryFileSystem {
/// The operation overrides the content for an existing file with the same normalized `path`.
///
/// Enclosing directories are automatically created if they don't exist.
pub fn write_file(
&self,
path: impl AsRef<FileSystemPath>,
content: impl ToString,
) -> Result<()> {
pub fn write_file(&self, path: impl AsRef<SystemPath>, content: impl ToString) -> Result<()> {
let mut by_path = self.inner.by_path.write().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
@ -95,26 +153,30 @@ impl MemoryFileSystem {
Ok(())
}
pub fn remove_file(&self, path: impl AsRef<FileSystemPath>) -> Result<()> {
let mut by_path = self.inner.by_path.write().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
pub fn remove_file(&self, path: impl AsRef<SystemPath>) -> Result<()> {
fn remove_file(fs: &MemoryFileSystem, path: &SystemPath) -> Result<()> {
let mut by_path = fs.inner.by_path.write().unwrap();
let normalized = normalize_path(path, &fs.inner.cwd);
match by_path.entry(normalized) {
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
Entry::File(_) => {
entry.remove();
Ok(())
}
Entry::Directory(_) => Err(is_a_directory()),
},
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
match by_path.entry(normalized) {
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
Entry::File(_) => {
entry.remove();
Ok(())
}
Entry::Directory(_) => Err(is_a_directory()),
},
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
}
}
remove_file(self, path.as_ref())
}
/// Sets the last modified timestamp of the file stored at `path` to now.
///
/// Creates a new file if the file at `path` doesn't exist.
pub fn touch(&self, path: impl AsRef<FileSystemPath>) -> Result<()> {
pub fn touch(&self, path: impl AsRef<SystemPath>) -> Result<()> {
let mut by_path = self.inner.by_path.write().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
@ -124,7 +186,7 @@ impl MemoryFileSystem {
}
/// Creates a directory at `path`. All enclosing directories are created if they don't exist.
pub fn create_directory_all(&self, path: impl AsRef<FileSystemPath>) -> Result<()> {
pub fn create_directory_all(&self, path: impl AsRef<SystemPath>) -> Result<()> {
let mut by_path = self.inner.by_path.write().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
@ -137,73 +199,34 @@ impl MemoryFileSystem {
/// * If the directory is not empty
/// * The `path` is not a directory
/// * The `path` does not exist
pub fn remove_directory(&self, path: impl AsRef<FileSystemPath>) -> Result<()> {
let mut by_path = self.inner.by_path.write().unwrap();
let normalized = normalize_path(path.as_ref(), &self.inner.cwd);
pub fn remove_directory(&self, path: impl AsRef<SystemPath>) -> Result<()> {
fn remove_directory(fs: &MemoryFileSystem, path: &SystemPath) -> Result<()> {
let mut by_path = fs.inner.by_path.write().unwrap();
let normalized = normalize_path(path, &fs.inner.cwd);
// Test if the directory is empty
// Skip the directory path itself
for (maybe_child, _) in by_path.range(normalized.clone()..).skip(1) {
if maybe_child.starts_with(&normalized) {
return Err(directory_not_empty());
} else if !maybe_child.as_str().starts_with(normalized.as_str()) {
break;
// Test if the directory is empty
// Skip the directory path itself
for (maybe_child, _) in by_path.range(normalized.clone()..).skip(1) {
if maybe_child.starts_with(&normalized) {
return Err(directory_not_empty());
} else if !maybe_child.as_str().starts_with(normalized.as_str()) {
break;
}
}
match by_path.entry(normalized.clone()) {
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
Entry::Directory(_) => {
entry.remove();
Ok(())
}
Entry::File(_) => Err(not_a_directory()),
},
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
}
}
match by_path.entry(normalized.clone()) {
std::collections::btree_map::Entry::Occupied(entry) => match entry.get() {
Entry::Directory(_) => {
entry.remove();
Ok(())
}
Entry::File(_) => Err(not_a_directory()),
},
std::collections::btree_map::Entry::Vacant(_) => Err(not_found()),
}
}
}
impl FileSystem for MemoryFileSystem {
fn metadata(&self, path: &FileSystemPath) -> Result<Metadata> {
let by_path = self.inner.by_path.read().unwrap();
let normalized = normalize_path(path, &self.inner.cwd);
let entry = by_path.get(&normalized).ok_or_else(not_found)?;
let metadata = match entry {
Entry::File(file) => Metadata {
revision: file.last_modified.into(),
permissions: Some(Self::PERMISSION),
file_type: FileType::File,
},
Entry::Directory(directory) => Metadata {
revision: directory.last_modified.into(),
permissions: Some(Self::PERMISSION),
file_type: FileType::Directory,
},
};
Ok(metadata)
}
fn read(&self, path: &FileSystemPath) -> Result<String> {
let by_path = self.inner.by_path.read().unwrap();
let normalized = normalize_path(path, &self.inner.cwd);
let entry = by_path.get(&normalized).ok_or_else(not_found)?;
match entry {
Entry::File(file) => Ok(file.content.clone()),
Entry::Directory(_) => Err(is_a_directory()),
}
}
fn exists(&self, path: &FileSystemPath) -> bool {
let by_path = self.inner.by_path.read().unwrap();
let normalized = normalize_path(path, &self.inner.cwd);
by_path.contains_key(&normalized)
remove_directory(self, path.as_ref())
}
}
@ -272,7 +295,7 @@ fn directory_not_empty() -> std::io::Error {
/// Normalizes the path by removing `.` and `..` components and transform the path into an absolute path.
///
/// Adapted from https://github.com/rust-lang/cargo/blob/fede83ccf973457de319ba6fa0e36ead454d2e20/src/cargo/util/paths.rs#L61
fn normalize_path(path: &FileSystemPath, cwd: &Utf8Path) -> Utf8PathBuf {
fn normalize_path(path: &SystemPath, cwd: &Utf8Path) -> Utf8PathBuf {
let path = camino::Utf8Path::new(path.as_str());
let mut components = path.components().peekable();
@ -353,14 +376,14 @@ mod tests {
use std::io::ErrorKind;
use std::time::Duration;
use crate::file_system::{FileSystem, FileSystemPath, MemoryFileSystem, Result};
use crate::system::{MemoryFileSystem, Result, SystemPath};
/// Creates a file system with the given files.
///
/// The content of all files will be empty.
fn with_files<P>(files: impl IntoIterator<Item = P>) -> super::MemoryFileSystem
where
P: AsRef<FileSystemPath>,
P: AsRef<SystemPath>,
{
let fs = MemoryFileSystem::new();
fs.write_files(files.into_iter().map(|path| (path, "")))
@ -371,7 +394,7 @@ mod tests {
#[test]
fn is_file() {
let path = FileSystemPath::new("a.py");
let path = SystemPath::new("a.py");
let fs = with_files([path]);
assert!(fs.is_file(path));
@ -382,26 +405,26 @@ mod tests {
fn exists() {
let fs = with_files(["a.py"]);
assert!(fs.exists(FileSystemPath::new("a.py")));
assert!(!fs.exists(FileSystemPath::new("b.py")));
assert!(fs.exists(SystemPath::new("a.py")));
assert!(!fs.exists(SystemPath::new("b.py")));
}
#[test]
fn exists_directories() {
let fs = with_files(["a/b/c.py"]);
assert!(fs.exists(FileSystemPath::new("a")));
assert!(fs.exists(FileSystemPath::new("a/b")));
assert!(fs.exists(FileSystemPath::new("a/b/c.py")));
assert!(fs.exists(SystemPath::new("a")));
assert!(fs.exists(SystemPath::new("a/b")));
assert!(fs.exists(SystemPath::new("a/b/c.py")));
}
#[test]
fn path_normalization() {
let fs = with_files(["a.py"]);
assert!(fs.exists(FileSystemPath::new("a.py")));
assert!(fs.exists(FileSystemPath::new("/a.py")));
assert!(fs.exists(FileSystemPath::new("/b/./../a.py")));
assert!(fs.exists(SystemPath::new("a.py")));
assert!(fs.exists(SystemPath::new("/a.py")));
assert!(fs.exists(SystemPath::new("/b/./../a.py")));
}
#[test]
@ -410,7 +433,7 @@ mod tests {
// The default permissions match the default on Linux: 0755
assert_eq!(
fs.metadata(FileSystemPath::new("a.py"))?.permissions(),
fs.metadata(SystemPath::new("a.py"))?.permissions(),
Some(MemoryFileSystem::PERMISSION)
);
@ -420,7 +443,7 @@ mod tests {
#[test]
fn touch() -> Result<()> {
let fs = MemoryFileSystem::new();
let path = FileSystemPath::new("a.py");
let path = SystemPath::new("a.py");
// Creates a file if it doesn't exist
fs.touch(path)?;
@ -445,16 +468,14 @@ mod tests {
fn create_dir_all() {
let fs = MemoryFileSystem::new();
fs.create_directory_all(FileSystemPath::new("a/b/c"))
.unwrap();
fs.create_directory_all(SystemPath::new("a/b/c")).unwrap();
assert!(fs.is_directory(FileSystemPath::new("a")));
assert!(fs.is_directory(FileSystemPath::new("a/b")));
assert!(fs.is_directory(FileSystemPath::new("a/b/c")));
assert!(fs.is_directory(SystemPath::new("a")));
assert!(fs.is_directory(SystemPath::new("a/b")));
assert!(fs.is_directory(SystemPath::new("a/b/c")));
// Should not fail if the directory already exists
fs.create_directory_all(FileSystemPath::new("a/b/c"))
.unwrap();
fs.create_directory_all(SystemPath::new("a/b/c")).unwrap();
}
#[test]
@ -462,7 +483,7 @@ mod tests {
let fs = with_files(["a/b.py"]);
let error = fs
.create_directory_all(FileSystemPath::new("a/b.py/c"))
.create_directory_all(SystemPath::new("a/b.py/c"))
.unwrap_err();
assert_eq!(error.kind(), ErrorKind::Other);
}
@ -472,7 +493,7 @@ mod tests {
let fs = with_files(["a/b.py"]);
let error = fs
.write_file(FileSystemPath::new("a/b.py/c"), "content".to_string())
.write_file(SystemPath::new("a/b.py/c"), "content".to_string())
.unwrap_err();
assert_eq!(error.kind(), ErrorKind::Other);
@ -485,7 +506,7 @@ mod tests {
fs.create_directory_all("a")?;
let error = fs
.write_file(FileSystemPath::new("a"), "content".to_string())
.write_file(SystemPath::new("a"), "content".to_string())
.unwrap_err();
assert_eq!(error.kind(), ErrorKind::Other);
@ -496,11 +517,11 @@ mod tests {
#[test]
fn read() -> Result<()> {
let fs = MemoryFileSystem::new();
let path = FileSystemPath::new("a.py");
let path = SystemPath::new("a.py");
fs.write_file(path, "Test content".to_string())?;
assert_eq!(fs.read(path)?, "Test content");
assert_eq!(fs.read_to_string(path)?, "Test content");
Ok(())
}
@ -511,7 +532,7 @@ mod tests {
fs.create_directory_all("a")?;
let error = fs.read(FileSystemPath::new("a")).unwrap_err();
let error = fs.read_to_string(SystemPath::new("a")).unwrap_err();
assert_eq!(error.kind(), ErrorKind::Other);
@ -522,7 +543,7 @@ mod tests {
fn read_fails_if_path_doesnt_exist() -> Result<()> {
let fs = MemoryFileSystem::new();
let error = fs.read(FileSystemPath::new("a")).unwrap_err();
let error = fs.read_to_string(SystemPath::new("a")).unwrap_err();
assert_eq!(error.kind(), ErrorKind::NotFound);
@ -535,13 +556,13 @@ mod tests {
fs.remove_file("a/a.py")?;
assert!(!fs.exists(FileSystemPath::new("a/a.py")));
assert!(!fs.exists(SystemPath::new("a/a.py")));
// It doesn't delete the enclosing directories
assert!(fs.exists(FileSystemPath::new("a")));
assert!(fs.exists(SystemPath::new("a")));
// It doesn't delete unrelated files.
assert!(fs.exists(FileSystemPath::new("b.py")));
assert!(fs.exists(SystemPath::new("b.py")));
Ok(())
}
@ -573,10 +594,10 @@ mod tests {
fs.remove_directory("a")?;
assert!(!fs.exists(FileSystemPath::new("a")));
assert!(!fs.exists(SystemPath::new("a")));
// It doesn't delete unrelated files.
assert!(fs.exists(FileSystemPath::new("b.py")));
assert!(fs.exists(SystemPath::new("b.py")));
Ok(())
}
@ -596,9 +617,9 @@ mod tests {
fs.remove_directory("foo").unwrap();
assert!(!fs.exists(FileSystemPath::new("foo")));
assert!(fs.exists(FileSystemPath::new("foo_bar.py")));
assert!(fs.exists(FileSystemPath::new("foob.py")));
assert!(!fs.exists(SystemPath::new("foo")));
assert!(fs.exists(SystemPath::new("foo_bar.py")));
assert!(fs.exists(SystemPath::new("foob.py")));
Ok(())
}

21
crates/ruff_db/src/file_system/os.rs → crates/ruff_db/src/system/os.rs
View file

@ -1,11 +1,12 @@
use filetime::FileTime;
use std::any::Any;
use crate::file_system::{FileSystem, FileSystemPath, FileType, Metadata, Result};
use crate::system::{FileType, Metadata, Result, System, SystemPath};
#[derive(Default, Debug)]
pub struct OsFileSystem;
pub struct OsSystem;
impl OsFileSystem {
impl OsSystem {
#[cfg(unix)]
fn permissions(metadata: &std::fs::Metadata) -> Option<u32> {
use std::os::unix::fs::PermissionsExt;
@ -23,8 +24,8 @@ impl OsFileSystem {
}
}
impl FileSystem for OsFileSystem {
fn metadata(&self, path: &FileSystemPath) -> Result<Metadata> {
impl System for OsSystem {
fn path_metadata(&self, path: &SystemPath) -> Result<Metadata> {
let metadata = path.as_std_path().metadata()?;
let last_modified = FileTime::from_last_modification_time(&metadata);
@ -35,13 +36,17 @@ impl FileSystem for OsFileSystem {
})
}
fn read(&self, path: &FileSystemPath) -> Result<String> {
std::fs::read_to_string(path)
fn read_to_string(&self, path: &SystemPath) -> Result<String> {
std::fs::read_to_string(path.as_std_path())
}
fn exists(&self, path: &FileSystemPath) -> bool {
fn path_exists(&self, path: &SystemPath) -> bool {
path.as_std_path().exists()
}
fn as_any(&self) -> &dyn Any {
self
}
}
impl From<std::fs::FileType> for FileType {

289
crates/ruff_db/src/file_system.rs → crates/ruff_db/src/system/path.rs
View file

@ -1,66 +1,25 @@
// TODO support untitled files for the LSP use case. Wrap a `str` and `String`
// The main question is how `as_std_path` would work for untitled files, that can only exist in the LSP case
// but there's no compile time guarantee that a [`OsSystem`] never gets an untitled file path.
use camino::{Utf8Path, Utf8PathBuf};
use std::fmt::Formatter;
use std::ops::Deref;
use std::path::{Path, StripPrefixError};
use camino::{Utf8Path, Utf8PathBuf};
use crate::file_revision::FileRevision;
pub use memory::MemoryFileSystem;
pub use os::OsFileSystem;
mod memory;
mod os;
pub type Result<T> = std::io::Result<T>;
/// An abstraction over `std::fs` with features tailored to Ruff's needs.
///
/// Provides a file system agnostic API to interact with files and directories.
/// Abstracting the file system operations enables:
///
/// * Accessing unsaved or even untitled files in the LSP use case
/// * Testing with an in-memory file system
/// * Running Ruff in a WASM environment without needing to stub out the full `std::fs` API.
pub trait FileSystem: std::fmt::Debug {
/// Reads the metadata of the file or directory at `path`.
fn metadata(&self, path: &FileSystemPath) -> Result<Metadata>;
/// Reads the content of the file at `path`.
fn read(&self, path: &FileSystemPath) -> Result<String>;
/// Returns `true` if `path` exists.
fn exists(&self, path: &FileSystemPath) -> bool;
/// Returns `true` if `path` exists and is a directory.
fn is_directory(&self, path: &FileSystemPath) -> bool {
self.metadata(path)
.map_or(false, |metadata| metadata.file_type.is_directory())
}
/// Returns `true` if `path` exists and is a file.
fn is_file(&self, path: &FileSystemPath) -> bool {
self.metadata(path)
.map_or(false, |metadata| metadata.file_type.is_file())
}
}
// TODO support untitled files for the LSP use case. Wrap a `str` and `String`
// The main question is how `as_std_path` would work for untitled files, that can only exist in the LSP case
// but there's no compile time guarantee that a [`OsFileSystem`] never gets an untitled file path.
/// Path to a file or directory stored in [`FileSystem`].
/// A slice of a path on [`System`](super::System) (akin to [`str`]).
///
/// The path is guaranteed to be valid UTF-8.
#[repr(transparent)]
#[derive(Eq, PartialEq, Hash, PartialOrd, Ord)]
pub struct FileSystemPath(Utf8Path);
pub struct SystemPath(Utf8Path);
impl FileSystemPath {
impl SystemPath {
pub fn new(path: &(impl AsRef<Utf8Path> + ?Sized)) -> &Self {
let path = path.as_ref();
// SAFETY: FsPath is marked as #[repr(transparent)] so the conversion from a
// *const Utf8Path to a *const FsPath is valid.
unsafe { &*(path as *const Utf8Path as *const FileSystemPath) }
unsafe { &*(path as *const Utf8Path as *const SystemPath) }
}
/// Extracts the file extension, if possible.
@ -75,10 +34,10 @@ impl FileSystemPath {
/// # Examples
///
/// ```
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// assert_eq!("rs", FileSystemPath::new("foo.rs").extension().unwrap());
/// assert_eq!("gz", FileSystemPath::new("foo.tar.gz").extension().unwrap());
/// assert_eq!("rs", SystemPath::new("foo.rs").extension().unwrap());
/// assert_eq!("gz", SystemPath::new("foo.tar.gz").extension().unwrap());
/// ```
///
/// See [`Path::extension`] for more details.
@ -95,9 +54,9 @@ impl FileSystemPath {
/// # Examples
///
/// ```
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// let path = FileSystemPath::new("/etc/passwd");
/// let path = SystemPath::new("/etc/passwd");
///
/// assert!(path.starts_with("/etc"));
/// assert!(path.starts_with("/etc/"));
@ -108,11 +67,11 @@ impl FileSystemPath {
/// assert!(!path.starts_with("/e"));
/// assert!(!path.starts_with("/etc/passwd.txt"));
///
/// assert!(!FileSystemPath::new("/etc/foo.rs").starts_with("/etc/foo"));
/// assert!(!SystemPath::new("/etc/foo.rs").starts_with("/etc/foo"));
/// ```
#[inline]
#[must_use]
pub fn starts_with(&self, base: impl AsRef<FileSystemPath>) -> bool {
pub fn starts_with(&self, base: impl AsRef<SystemPath>) -> bool {
self.0.starts_with(base.as_ref())
}
@ -123,9 +82,9 @@ impl FileSystemPath {
/// # Examples
///
/// ```
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// let path = FileSystemPath::new("/etc/resolv.conf");
/// let path = SystemPath::new("/etc/resolv.conf");
///
/// assert!(path.ends_with("resolv.conf"));
/// assert!(path.ends_with("etc/resolv.conf"));
@ -136,7 +95,7 @@ impl FileSystemPath {
/// ```
#[inline]
#[must_use]
pub fn ends_with(&self, child: impl AsRef<FileSystemPath>) -> bool {
pub fn ends_with(&self, child: impl AsRef<SystemPath>) -> bool {
self.0.ends_with(child.as_ref())
}
@ -147,20 +106,20 @@ impl FileSystemPath {
/// # Examples
///
/// ```
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// let path = FileSystemPath::new("/foo/bar");
/// let path = SystemPath::new("/foo/bar");
/// let parent = path.parent().unwrap();
/// assert_eq!(parent, FileSystemPath::new("/foo"));
/// assert_eq!(parent, SystemPath::new("/foo"));
///
/// let grand_parent = parent.parent().unwrap();
/// assert_eq!(grand_parent, FileSystemPath::new("/"));
/// assert_eq!(grand_parent, SystemPath::new("/"));
/// assert_eq!(grand_parent.parent(), None);
/// ```
#[inline]
#[must_use]
pub fn parent(&self) -> Option<&FileSystemPath> {
self.0.parent().map(FileSystemPath::new)
pub fn parent(&self) -> Option<&SystemPath> {
self.0.parent().map(SystemPath::new)
}
/// Produces an iterator over the [`camino::Utf8Component`]s of the path.
@ -185,9 +144,9 @@ impl FileSystemPath {
///
/// ```
/// use camino::{Utf8Component};
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// let mut components = FileSystemPath::new("/tmp/foo.txt").components();
/// let mut components = SystemPath::new("/tmp/foo.txt").components();
///
/// assert_eq!(components.next(), Some(Utf8Component::RootDir));
/// assert_eq!(components.next(), Some(Utf8Component::Normal("tmp")));
@ -212,14 +171,14 @@ impl FileSystemPath {
///
/// ```
/// use camino::Utf8Path;
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// assert_eq!(Some("bin"), FileSystemPath::new("/usr/bin/").file_name());
/// assert_eq!(Some("foo.txt"), FileSystemPath::new("tmp/foo.txt").file_name());
/// assert_eq!(Some("foo.txt"), FileSystemPath::new("foo.txt/.").file_name());
/// assert_eq!(Some("foo.txt"), FileSystemPath::new("foo.txt/.//").file_name());
/// assert_eq!(None, FileSystemPath::new("foo.txt/..").file_name());
/// assert_eq!(None, FileSystemPath::new("/").file_name());
/// assert_eq!(Some("bin"), SystemPath::new("/usr/bin/").file_name());
/// assert_eq!(Some("foo.txt"), SystemPath::new("tmp/foo.txt").file_name());
/// assert_eq!(Some("foo.txt"), SystemPath::new("foo.txt/.").file_name());
/// assert_eq!(Some("foo.txt"), SystemPath::new("foo.txt/.//").file_name());
/// assert_eq!(None, SystemPath::new("foo.txt/..").file_name());
/// assert_eq!(None, SystemPath::new("/").file_name());
/// ```
#[inline]
#[must_use]
@ -229,7 +188,7 @@ impl FileSystemPath {
/// Extracts the stem (non-extension) portion of [`self.file_name`].
///
/// [`self.file_name`]: FileSystemPath::file_name
/// [`self.file_name`]: SystemPath::file_name
///
/// The stem is:
///
@ -241,10 +200,10 @@ impl FileSystemPath {
/// # Examples
///
/// ```
/// use ruff_db::file_system::FileSystemPath;
/// use ruff_db::system::SystemPath;
///
/// assert_eq!("foo", FileSystemPath::new("foo.rs").file_stem().unwrap());
/// assert_eq!("foo.tar", FileSystemPath::new("foo.tar.gz").file_stem().unwrap());
/// assert_eq!("foo", SystemPath::new("foo.rs").file_stem().unwrap());
/// assert_eq!("foo.tar", SystemPath::new("foo.tar.gz").file_stem().unwrap());
/// ```
#[inline]
#[must_use]
@ -259,77 +218,77 @@ impl FileSystemPath {
/// If `base` is not a prefix of `self` (i.e., [`starts_with`]
/// returns `false`), returns [`Err`].
///
/// [`starts_with`]: FileSystemPath::starts_with
/// [`starts_with`]: SystemPath::starts_with
///
/// # Examples
///
/// ```
/// use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf};
/// use ruff_db::system::{SystemPath, SystemPathBuf};
///
/// let path = FileSystemPath::new("/test/haha/foo.txt");
/// let path = SystemPath::new("/test/haha/foo.txt");
///
/// assert_eq!(path.strip_prefix("/"), Ok(FileSystemPath::new("test/haha/foo.txt")));
/// assert_eq!(path.strip_prefix("/test"), Ok(FileSystemPath::new("haha/foo.txt")));
/// assert_eq!(path.strip_prefix("/test/"), Ok(FileSystemPath::new("haha/foo.txt")));
/// assert_eq!(path.strip_prefix("/test/haha/foo.txt"), Ok(FileSystemPath::new("")));
/// assert_eq!(path.strip_prefix("/test/haha/foo.txt/"), Ok(FileSystemPath::new("")));
/// assert_eq!(path.strip_prefix("/"), Ok(SystemPath::new("test/haha/foo.txt")));
/// assert_eq!(path.strip_prefix("/test"), Ok(SystemPath::new("haha/foo.txt")));
/// assert_eq!(path.strip_prefix("/test/"), Ok(SystemPath::new("haha/foo.txt")));
/// assert_eq!(path.strip_prefix("/test/haha/foo.txt"), Ok(SystemPath::new("")));
/// assert_eq!(path.strip_prefix("/test/haha/foo.txt/"), Ok(SystemPath::new("")));
///
/// assert!(path.strip_prefix("test").is_err());
/// assert!(path.strip_prefix("/haha").is_err());
///
/// let prefix = FileSystemPathBuf::from("/test/");
/// assert_eq!(path.strip_prefix(prefix), Ok(FileSystemPath::new("haha/foo.txt")));
/// let prefix = SystemPathBuf::from("/test/");
/// assert_eq!(path.strip_prefix(prefix), Ok(SystemPath::new("haha/foo.txt")));
/// ```
#[inline]
pub fn strip_prefix(
&self,
base: impl AsRef<FileSystemPath>,
) -> std::result::Result<&FileSystemPath, StripPrefixError> {
self.0.strip_prefix(base.as_ref()).map(FileSystemPath::new)
base: impl AsRef<SystemPath>,
) -> std::result::Result<&SystemPath, StripPrefixError> {
self.0.strip_prefix(base.as_ref()).map(SystemPath::new)
}
/// Creates an owned [`FileSystemPathBuf`] with `path` adjoined to `self`.
/// Creates an owned [`SystemPathBuf`] with `path` adjoined to `self`.
///
/// See [`std::path::PathBuf::push`] for more details on what it means to adjoin a path.
///
/// # Examples
///
/// ```
/// use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf};
/// use ruff_db::system::{SystemPath, SystemPathBuf};
///
/// assert_eq!(FileSystemPath::new("/etc").join("passwd"), FileSystemPathBuf::from("/etc/passwd"));
/// assert_eq!(SystemPath::new("/etc").join("passwd"), SystemPathBuf::from("/etc/passwd"));
/// ```
#[inline]
#[must_use]
pub fn join(&self, path: impl AsRef<FileSystemPath>) -> FileSystemPathBuf {
FileSystemPathBuf::from_utf8_path_buf(self.0.join(&path.as_ref().0))
pub fn join(&self, path: impl AsRef<SystemPath>) -> SystemPathBuf {
SystemPathBuf::from_utf8_path_buf(self.0.join(&path.as_ref().0))
}
/// Creates an owned [`FileSystemPathBuf`] like `self` but with the given extension.
/// Creates an owned [`SystemPathBuf`] like `self` but with the given extension.
///
/// See [`std::path::PathBuf::set_extension`] for more details.
///
/// # Examples
///
/// ```
/// use ruff_db::file_system::{FileSystemPath, FileSystemPathBuf};
/// use ruff_db::system::{SystemPath, SystemPathBuf};
///
/// let path = FileSystemPath::new("foo.rs");
/// assert_eq!(path.with_extension("txt"), FileSystemPathBuf::from("foo.txt"));
/// let path = SystemPath::new("foo.rs");
/// assert_eq!(path.with_extension("txt"), SystemPathBuf::from("foo.txt"));
///
/// let path = FileSystemPath::new("foo.tar.gz");
/// assert_eq!(path.with_extension(""), FileSystemPathBuf::from("foo.tar"));
/// assert_eq!(path.with_extension("xz"), FileSystemPathBuf::from("foo.tar.xz"));
/// assert_eq!(path.with_extension("").with_extension("txt"), FileSystemPathBuf::from("foo.txt"));
/// let path = SystemPath::new("foo.tar.gz");
/// assert_eq!(path.with_extension(""), SystemPathBuf::from("foo.tar"));
/// assert_eq!(path.with_extension("xz"), SystemPathBuf::from("foo.tar.xz"));
/// assert_eq!(path.with_extension("").with_extension("txt"), SystemPathBuf::from("foo.txt"));
/// ```
#[inline]
pub fn with_extension(&self, extension: &str) -> FileSystemPathBuf {
FileSystemPathBuf::from_utf8_path_buf(self.0.with_extension(extension))
pub fn with_extension(&self, extension: &str) -> SystemPathBuf {
SystemPathBuf::from_utf8_path_buf(self.0.with_extension(extension))
}
/// Converts the path to an owned [`FileSystemPathBuf`].
pub fn to_path_buf(&self) -> FileSystemPathBuf {
FileSystemPathBuf(self.0.to_path_buf())
/// Converts the path to an owned [`SystemPathBuf`].
pub fn to_path_buf(&self) -> SystemPathBuf {
SystemPathBuf(self.0.to_path_buf())
}
/// Returns the path as a string slice.
@ -344,19 +303,19 @@ impl FileSystemPath {
self.0.as_std_path()
}
pub fn from_std_path(path: &Path) -> Option<&FileSystemPath> {
Some(FileSystemPath::new(Utf8Path::from_path(path)?))
pub fn from_std_path(path: &Path) -> Option<&SystemPath> {
Some(SystemPath::new(Utf8Path::from_path(path)?))
}
}
/// Owned path to a file or directory stored in [`FileSystem`].
/// An owned, mutable path on [`System`](`super::System`) (akin to [`String`]).
///
/// The path is guaranteed to be valid UTF-8.
#[repr(transparent)]
#[derive(Eq, PartialEq, Clone, Hash, PartialOrd, Ord)]
pub struct FileSystemPathBuf(Utf8PathBuf);
pub struct SystemPathBuf(Utf8PathBuf);
impl FileSystemPathBuf {
impl SystemPathBuf {
pub fn new() -> Self {
Self(Utf8PathBuf::new())
}
@ -386,82 +345,82 @@ impl FileSystemPathBuf {
/// Pushing a relative path extends the existing path:
///
/// ```
/// use ruff_db::file_system::FileSystemPathBuf;
/// use ruff_db::system::SystemPathBuf;
///
/// let mut path = FileSystemPathBuf::from("/tmp");
/// let mut path = SystemPathBuf::from("/tmp");
/// path.push("file.bk");
/// assert_eq!(path, FileSystemPathBuf::from("/tmp/file.bk"));
/// assert_eq!(path, SystemPathBuf::from("/tmp/file.bk"));
/// ```
///
/// Pushing an absolute path replaces the existing path:
///
/// ```
///
/// use ruff_db::file_system::FileSystemPathBuf;
/// use ruff_db::system::SystemPathBuf;
///
/// let mut path = FileSystemPathBuf::from("/tmp");
/// let mut path = SystemPathBuf::from("/tmp");
/// path.push("/etc");
/// assert_eq!(path, FileSystemPathBuf::from("/etc"));
/// assert_eq!(path, SystemPathBuf::from("/etc"));
/// ```
pub fn push(&mut self, path: impl AsRef<FileSystemPath>) {
pub fn push(&mut self, path: impl AsRef<SystemPath>) {
self.0.push(&path.as_ref().0);
}
#[inline]
pub fn as_path(&self) -> &FileSystemPath {
FileSystemPath::new(&self.0)
pub fn as_path(&self) -> &SystemPath {
SystemPath::new(&self.0)
}
}
impl From<&str> for FileSystemPathBuf {
impl From<&str> for SystemPathBuf {
fn from(value: &str) -> Self {
FileSystemPathBuf::from_utf8_path_buf(Utf8PathBuf::from(value))
SystemPathBuf::from_utf8_path_buf(Utf8PathBuf::from(value))
}
}
impl Default for FileSystemPathBuf {
impl Default for SystemPathBuf {
fn default() -> Self {
Self::new()
}
}
impl AsRef<FileSystemPath> for FileSystemPathBuf {
impl AsRef<SystemPath> for SystemPathBuf {
#[inline]
fn as_ref(&self) -> &FileSystemPath {
fn as_ref(&self) -> &SystemPath {
self.as_path()
}
}
impl AsRef<FileSystemPath> for FileSystemPath {
impl AsRef<SystemPath> for SystemPath {
#[inline]
fn as_ref(&self) -> &FileSystemPath {
fn as_ref(&self) -> &SystemPath {
self
}
}
impl AsRef<FileSystemPath> for str {
impl AsRef<SystemPath> for str {
#[inline]
fn as_ref(&self) -> &FileSystemPath {
FileSystemPath::new(self)
fn as_ref(&self) -> &SystemPath {
SystemPath::new(self)
}
}
impl AsRef<FileSystemPath> for String {
impl AsRef<SystemPath> for String {
#[inline]
fn as_ref(&self) -> &FileSystemPath {
FileSystemPath::new(self)
fn as_ref(&self) -> &SystemPath {
SystemPath::new(self)
}
}
impl AsRef<Path> for FileSystemPath {
impl AsRef<Path> for SystemPath {
#[inline]
fn as_ref(&self) -> &Path {
self.0.as_std_path()
}
}
impl Deref for FileSystemPathBuf {
type Target = FileSystemPath;
impl Deref for SystemPathBuf {
type Target = SystemPath;
#[inline]
fn deref(&self) -> &Self::Target {
@ -469,68 +428,26 @@ impl Deref for FileSystemPathBuf {
}
}
impl std::fmt::Debug for FileSystemPath {
impl std::fmt::Debug for SystemPath {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::fmt::Display for FileSystemPath {
impl std::fmt::Display for SystemPath {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::fmt::Debug for FileSystemPathBuf {
impl std::fmt::Debug for SystemPathBuf {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::fmt::Display for FileSystemPathBuf {
impl std::fmt::Display for SystemPathBuf {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Metadata {
revision: FileRevision,
permissions: Option<u32>,
file_type: FileType,
}
impl Metadata {
pub fn revision(&self) -> FileRevision {
self.revision
}
pub fn permissions(&self) -> Option<u32> {
self.permissions
}
pub fn file_type(&self) -> FileType {
self.file_type
}
}
#[derive(Copy, Clone, Eq, PartialEq, Debug, Hash)]
pub enum FileType {
File,
Directory,
Symlink,
}
impl FileType {
pub const fn is_file(self) -> bool {
matches!(self, FileType::File)
}
pub const fn is_directory(self) -> bool {
matches!(self, FileType::Directory)
}
pub const fn is_symlink(self) -> bool {
matches!(self, FileType::Symlink)
}
}

167
crates/ruff_db/src/system/test.rs Normal file
View file

@ -0,0 +1,167 @@
use crate::files::{File, FilePath};
use crate::system::{MemoryFileSystem, Metadata, OsSystem, System, SystemPath};
use crate::Db;
use std::any::Any;
/// System implementation intended for testing.
///
/// It uses a memory-file system by default, but can be switched to the real file system for tests
/// verifying more advanced file system features.
///
/// ## Warning
/// Don't use this system for production code. It's intended for testing only.
#[derive(Default, Debug)]
pub struct TestSystem {
inner: TestFileSystem,
}
impl TestSystem {
pub fn snapshot(&self) -> Self {
Self {
inner: self.inner.snapshot(),
}
}
/// Returns the memory file system.
///
/// ## Panics
/// If this test db isn't using a memory file system.
pub fn memory_file_system(&self) -> &MemoryFileSystem {
if let TestFileSystem::Stub(fs) = &self.inner {
fs
} else {
panic!("The test db is not using a memory file system");
}
}
fn use_os_system(&mut self) {
self.inner = TestFileSystem::Os(OsSystem);
}
}
impl System for TestSystem {
fn path_metadata(&self, path: &SystemPath) -> crate::system::Result<Metadata> {
match &self.inner {
TestFileSystem::Stub(fs) => fs.metadata(path),
TestFileSystem::Os(fs) => fs.path_metadata(path),
}
}
fn read_to_string(&self, path: &SystemPath) -> crate::system::Result<String> {
match &self.inner {
TestFileSystem::Stub(fs) => fs.read_to_string(path),
TestFileSystem::Os(fs) => fs.read_to_string(path),
}
}
fn path_exists(&self, path: &SystemPath) -> bool {
match &self.inner {
TestFileSystem::Stub(fs) => fs.exists(path),
TestFileSystem::Os(fs) => fs.path_exists(path),
}
}
fn is_directory(&self, path: &SystemPath) -> bool {
match &self.inner {
TestFileSystem::Stub(fs) => fs.is_directory(path),
TestFileSystem::Os(fs) => fs.is_directory(path),
}
}
fn is_file(&self, path: &SystemPath) -> bool {
match &self.inner {
TestFileSystem::Stub(fs) => fs.is_file(path),
TestFileSystem::Os(fs) => fs.is_file(path),
}
}
fn as_any(&self) -> &dyn Any {
self
}
}
/// Extension trait for databases that use [`TestSystem`].
///
/// Provides various helper function that ease testing.
pub trait DbWithTestSystem: Db + Sized {
fn test_system(&self) -> &TestSystem;
fn test_system_mut(&mut self) -> &mut TestSystem;
/// Writes the content of the given file and notifies the Db about the change.
///
/// # Panics
/// If the system isn't using the memory file system.
fn write_file(
&mut self,
path: impl AsRef<SystemPath>,
content: impl ToString,
) -> crate::system::Result<()> {
let path = path.as_ref().to_path_buf();
let result = self
.test_system()
.memory_file_system()
.write_file(&path, content);
if result.is_ok() {
File::touch_path(self, &FilePath::System(path));
}
result
}
/// Writes the content of the given file and notifies the Db about the change.
///
/// # Panics
/// If the system isn't using the memory file system for testing.
fn write_files<P, C, I>(&mut self, files: I) -> crate::system::Result<()>
where
I: IntoIterator<Item = (P, C)>,
P: AsRef<SystemPath>,
C: ToString,
{
for (path, content) in files {
self.write_file(path, content)?;
}
Ok(())
}
/// Uses the real file system instead of the memory file system.
///
/// This useful for testing advanced file system features like permissions, symlinks, etc.
///
/// Note that any files written to the memory file system won't be copied over.
fn use_os_system(&mut self) {
self.test_system_mut().use_os_system();
}
/// Returns the memory file system.
///
/// ## Panics
/// If this system isn't using a memory file system.
fn memory_file_system(&self) -> &MemoryFileSystem {
self.test_system().memory_file_system()
}
}
#[derive(Debug)]
enum TestFileSystem {
Stub(MemoryFileSystem),
Os(OsSystem),
}
impl TestFileSystem {
fn snapshot(&self) -> Self {
match self {
Self::Stub(fs) => Self::Stub(fs.snapshot()),
Self::Os(fs) => Self::Os(fs.snapshot()),
}
}
}
impl Default for TestFileSystem {
fn default() -> Self {
Self::Stub(MemoryFileSystem::default())
}
}

225
crates/ruff_db/src/vendored.rs
View file

@ -2,14 +2,15 @@ use std::borrow::Cow;
use std::collections::BTreeMap;
use std::fmt::{self, Debug};
use std::io::{self, Read};
use std::sync::{Mutex, MutexGuard};
use std::sync::{Arc, Mutex, MutexGuard};
use zip::{read::ZipFile, ZipArchive};
use zip::{read::ZipFile, ZipArchive, ZipWriter};
use crate::file_revision::FileRevision;
pub use path::{VendoredPath, VendoredPathBuf};
pub mod path;
pub use self::path::{VendoredPath, VendoredPathBuf};
mod path;
type Result<T> = io::Result<T>;
type LockedZipArchive<'a> = MutexGuard<'a, VendoredZipArchive>;
@ -20,46 +21,75 @@ type LockedZipArchive<'a> = MutexGuard<'a, VendoredZipArchive>;
/// "Files" in the `VendoredFileSystem` are read-only and immutable.
/// Directories are supported, but symlinks and hardlinks cannot exist.
pub struct VendoredFileSystem {
inner: Mutex<VendoredZipArchive>,
inner: Arc<Mutex<VendoredZipArchive>>,
}
impl VendoredFileSystem {
pub fn new(raw_bytes: &'static [u8]) -> Result<Self> {
pub fn new_static(raw_bytes: &'static [u8]) -> Result<Self> {
Self::new_impl(Cow::Borrowed(raw_bytes))
}
pub fn new(raw_bytes: Vec<u8>) -> Result<Self> {
Self::new_impl(Cow::Owned(raw_bytes))
}
fn new_impl(data: Cow<'static, [u8]>) -> Result<Self> {
Ok(Self {
inner: Mutex::new(VendoredZipArchive::new(raw_bytes)?),
inner: Arc::new(Mutex::new(VendoredZipArchive::new(data)?)),
})
}
pub fn exists(&self, path: &VendoredPath) -> bool {
let normalized = NormalizedVendoredPath::from(path);
let mut archive = self.lock_archive();
// Must probe the zipfile twice, as "stdlib" and "stdlib/" are considered
// different paths in a zip file, but we want to abstract over that difference here
// so that paths relative to the `VendoredFileSystem`
// work the same as other paths in Ruff.
archive.lookup_path(&normalized).is_ok()
|| archive
.lookup_path(&normalized.with_trailing_slash())
.is_ok()
pub fn snapshot(&self) -> Self {
Self {
inner: Arc::clone(&self.inner),
}
}
pub fn metadata(&self, path: &VendoredPath) -> Option<Metadata> {
let normalized = NormalizedVendoredPath::from(path);
let mut archive = self.lock_archive();
pub fn exists(&self, path: impl AsRef<VendoredPath>) -> bool {
fn exists(fs: &VendoredFileSystem, path: &VendoredPath) -> bool {
let normalized = NormalizedVendoredPath::from(path);
let mut archive = fs.lock_archive();
// Must probe the zipfile twice, as "stdlib" and "stdlib/" are considered
// different paths in a zip file, but we want to abstract over that difference here
// so that paths relative to the `VendoredFileSystem`
// work the same as other paths in Ruff.
if let Ok(zip_file) = archive.lookup_path(&normalized) {
return Some(Metadata::from_zip_file(zip_file));
}
if let Ok(zip_file) = archive.lookup_path(&normalized.with_trailing_slash()) {
return Some(Metadata::from_zip_file(zip_file));
// Must probe the zipfile twice, as "stdlib" and "stdlib/" are considered
// different paths in a zip file, but we want to abstract over that difference here
// so that paths relative to the `VendoredFileSystem`
// work the same as other paths in Ruff.
archive.lookup_path(&normalized).is_ok()
|| archive
.lookup_path(&normalized.with_trailing_slash())
.is_ok()
}
None
exists(self, path.as_ref())
}
pub fn metadata(&self, path: impl AsRef<VendoredPath>) -> Result<Metadata> {
fn metadata(fs: &VendoredFileSystem, path: &VendoredPath) -> Result<Metadata> {
let normalized = NormalizedVendoredPath::from(path);
let mut archive = fs.lock_archive();
// Must probe the zipfile twice, as "stdlib" and "stdlib/" are considered
// different paths in a zip file, but we want to abstract over that difference here
// so that paths relative to the `VendoredFileSystem`
// work the same as other paths in Ruff.
if let Ok(zip_file) = archive.lookup_path(&normalized) {
return Ok(Metadata::from_zip_file(zip_file));
}
let zip_file = archive.lookup_path(&normalized.with_trailing_slash())?;
Ok(Metadata::from_zip_file(zip_file))
}
metadata(self, path.as_ref())
}
pub fn is_directory(&self, path: impl AsRef<VendoredPath>) -> bool {
self.metadata(path)
.is_ok_and(|metadata| metadata.kind().is_directory())
}
pub fn is_file(&self, path: impl AsRef<VendoredPath>) -> bool {
self.metadata(path)
.is_ok_and(|metadata| metadata.kind().is_file())
}
/// Read the entire contents of the zip file at `path` into a string
@ -68,12 +98,16 @@ impl VendoredFileSystem {
/// - The path does not exist in the underlying zip archive
/// - The path exists in the underlying zip archive, but represents a directory
/// - The contents of the zip file at `path` contain invalid UTF-8
pub fn read(&self, path: &VendoredPath) -> Result<String> {
let mut archive = self.lock_archive();
let mut zip_file = archive.lookup_path(&NormalizedVendoredPath::from(path))?;
let mut buffer = String::new();
zip_file.read_to_string(&mut buffer)?;
Ok(buffer)
pub fn read_to_string(&self, path: impl AsRef<VendoredPath>) -> Result<String> {
fn read_to_string(fs: &VendoredFileSystem, path: &VendoredPath) -> Result<String> {
let mut archive = fs.lock_archive();
let mut zip_file = archive.lookup_path(&NormalizedVendoredPath::from(path))?;
let mut buffer = String::new();
zip_file.read_to_string(&mut buffer)?;
Ok(buffer)
}
read_to_string(self, path.as_ref())
}
/// Acquire a lock on the underlying zip archive.
@ -112,6 +146,20 @@ impl fmt::Debug for VendoredFileSystem {
}
}
impl Default for VendoredFileSystem {
fn default() -> Self {
let mut bytes: Vec<u8> = Vec::new();
let mut cursor = io::Cursor::new(&mut bytes);
{
let mut writer = ZipWriter::new(&mut cursor);
writer.finish().unwrap();
}
VendoredFileSystem::new(bytes).unwrap()
}
}
/// Private struct only used in `Debug` implementations
///
/// This could possibly be unified with the `Metadata` struct,
@ -195,10 +243,10 @@ impl Metadata {
/// Newtype wrapper around a ZipArchive.
#[derive(Debug)]
struct VendoredZipArchive(ZipArchive<io::Cursor<&'static [u8]>>);
struct VendoredZipArchive(ZipArchive<io::Cursor<Cow<'static, [u8]>>>);
impl VendoredZipArchive {
fn new(data: &'static [u8]) -> Result<Self> {
fn new(data: Cow<'static, [u8]>) -> Result<Self> {
Ok(Self(ZipArchive::new(io::Cursor::new(data))?))
}
@ -290,11 +338,11 @@ impl<'a> From<&'a VendoredPath> for NormalizedVendoredPath<'a> {
}
#[cfg(test)]
mod tests {
pub(crate) mod tests {
use std::io::Write;
use insta::assert_snapshot;
use once_cell::sync::Lazy;
use zip::result::ZipResult;
use zip::write::FileOptions;
use zip::{CompressionMethod, ZipWriter};
@ -303,37 +351,66 @@ mod tests {
const FUNCTOOLS_CONTENTS: &str = "def update_wrapper(): ...";
const ASYNCIO_TASKS_CONTENTS: &str = "class Task: ...";
static MOCK_ZIP_ARCHIVE: Lazy<Box<[u8]>> = Lazy::new(|| {
let mut typeshed_buffer = Vec::new();
let typeshed = io::Cursor::new(&mut typeshed_buffer);
pub struct VendoredFileSystemBuilder {
writer: ZipWriter<io::Cursor<Vec<u8>>>,
}
let options = FileOptions::default()
.compression_method(CompressionMethod::Zstd)
.unix_permissions(0o644);
impl Default for VendoredFileSystemBuilder {
fn default() -> Self {
Self::new()
}
}
{
let mut archive = ZipWriter::new(typeshed);
impl VendoredFileSystemBuilder {
pub fn new() -> Self {
let buffer = io::Cursor::new(Vec::new());
archive.add_directory("stdlib/", options).unwrap();
archive.start_file("stdlib/functools.pyi", options).unwrap();
archive.write_all(FUNCTOOLS_CONTENTS.as_bytes()).unwrap();
archive.add_directory("stdlib/asyncio/", options).unwrap();
archive
.start_file("stdlib/asyncio/tasks.pyi", options)
.unwrap();
archive
.write_all(ASYNCIO_TASKS_CONTENTS.as_bytes())
.unwrap();
archive.finish().unwrap();
Self {
writer: ZipWriter::new(buffer),
}
}
typeshed_buffer.into_boxed_slice()
});
pub fn add_file(
&mut self,
path: impl AsRef<VendoredPath>,
content: &str,
) -> std::io::Result<()> {
self.writer
.start_file(path.as_ref().as_str(), Self::options())?;
self.writer.write_all(content.as_bytes())
}
pub fn add_directory(&mut self, path: impl AsRef<VendoredPath>) -> ZipResult<()> {
self.writer
.add_directory(path.as_ref().as_str(), Self::options())
}
pub fn finish(mut self) -> Result<VendoredFileSystem> {
let buffer = self.writer.finish()?;
VendoredFileSystem::new(buffer.into_inner())
}
fn options() -> FileOptions {
FileOptions::default()
.compression_method(CompressionMethod::Zstd)
.unix_permissions(0o644)
}
}
fn mock_typeshed() -> VendoredFileSystem {
VendoredFileSystem::new(&MOCK_ZIP_ARCHIVE).unwrap()
let mut builder = VendoredFileSystemBuilder::new();
builder.add_directory("stdlib/").unwrap();
builder
.add_file("stdlib/functools.pyi", FUNCTOOLS_CONTENTS)
.unwrap();
builder.add_directory("stdlib/asyncio/").unwrap();
builder
.add_file("stdlib/asyncio/tasks.pyi", ASYNCIO_TASKS_CONTENTS)
.unwrap();
builder.finish().unwrap()
}
#[test]
@ -395,9 +472,9 @@ mod tests {
let path = VendoredPath::new(dirname);
assert!(mock_typeshed.exists(path));
assert!(mock_typeshed.read(path).is_err());
assert!(mock_typeshed.read_to_string(path).is_err());
let metadata = mock_typeshed.metadata(path).unwrap();
assert!(metadata.kind.is_directory());
assert!(metadata.kind().is_directory());
}
#[test]
@ -434,9 +511,9 @@ mod tests {
let mock_typeshed = mock_typeshed();
let path = VendoredPath::new(path);
assert!(!mock_typeshed.exists(path));
assert!(mock_typeshed.metadata(path).is_none());
assert!(mock_typeshed.metadata(path).is_err());
assert!(mock_typeshed
.read(path)
.read_to_string(path)
.is_err_and(|err| err.to_string().contains("file not found")));
}
@ -463,7 +540,7 @@ mod tests {
fn test_file(mock_typeshed: &VendoredFileSystem, path: &VendoredPath) {
assert!(mock_typeshed.exists(path));
let metadata = mock_typeshed.metadata(path).unwrap();
assert!(metadata.kind.is_file());
assert!(metadata.kind().is_file());
}
#[test]
@ -471,11 +548,11 @@ mod tests {
let mock_typeshed = mock_typeshed();
let path = VendoredPath::new("stdlib/functools.pyi");
test_file(&mock_typeshed, path);
let functools_stub = mock_typeshed.read(path).unwrap();
let functools_stub = mock_typeshed.read_to_string(path).unwrap();
assert_eq!(functools_stub.as_str(), FUNCTOOLS_CONTENTS);
// Test that using the RefCell doesn't mutate
// the internal state of the underlying zip archive incorrectly:
let functools_stub_again = mock_typeshed.read(path).unwrap();
let functools_stub_again = mock_typeshed.read_to_string(path).unwrap();
assert_eq!(functools_stub_again.as_str(), FUNCTOOLS_CONTENTS);
}
@ -492,7 +569,7 @@ mod tests {
let mock_typeshed = mock_typeshed();
let path = VendoredPath::new("stdlib/asyncio/tasks.pyi");
test_file(&mock_typeshed, path);
let asyncio_stub = mock_typeshed.read(path).unwrap();
let asyncio_stub = mock_typeshed.read_to_string(path).unwrap();
assert_eq!(asyncio_stub.as_str(), ASYNCIO_TASKS_CONTENTS);
}

161
crates/ruff_db/src/vfs/path.rs
View file

@ -1,161 +0,0 @@
use crate::file_system::{FileSystemPath, FileSystemPathBuf};
use crate::vendored::path::{VendoredPath, VendoredPathBuf};
/// Path to a file.
///
/// The path abstracts that files in Ruff can come from different sources:
///
/// * a file stored on disk
/// * a vendored file that ships as part of the ruff binary
/// * Future: A virtual file that references a slice of another file. For example, the CSS code in a python file.
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
pub enum VfsPath {
/// Path that points to a file on disk.
FileSystem(FileSystemPathBuf),
Vendored(VendoredPathBuf),
}
impl VfsPath {
/// Create a new path to a file on the file system.
#[must_use]
pub fn file_system(path: impl AsRef<FileSystemPath>) -> Self {
VfsPath::FileSystem(path.as_ref().to_path_buf())
}
/// Returns `Some` if the path is a file system path that points to a path on disk.
#[must_use]
#[inline]
pub fn into_file_system_path_buf(self) -> Option<FileSystemPathBuf> {
match self {
VfsPath::FileSystem(path) => Some(path),
VfsPath::Vendored(_) => None,
}
}
#[must_use]
#[inline]
pub fn as_file_system_path(&self) -> Option<&FileSystemPath> {
match self {
VfsPath::FileSystem(path) => Some(path.as_path()),
VfsPath::Vendored(_) => None,
}
}
/// Returns `true` if the path is a file system path that points to a path on disk.
#[must_use]
#[inline]
pub const fn is_file_system_path(&self) -> bool {
matches!(self, VfsPath::FileSystem(_))
}
/// Returns `true` if the path is a vendored path.
#[must_use]
#[inline]
pub const fn is_vendored_path(&self) -> bool {
matches!(self, VfsPath::Vendored(_))
}
#[must_use]
#[inline]
pub fn as_vendored_path(&self) -> Option<&VendoredPath> {
match self {
VfsPath::Vendored(path) => Some(path.as_path()),
VfsPath::FileSystem(_) => None,
}
}
/// Yields the underlying [`str`] slice.
pub fn as_str(&self) -> &str {
match self {
VfsPath::FileSystem(path) => path.as_str(),
VfsPath::Vendored(path) => path.as_str(),
}
}
}
impl AsRef<str> for VfsPath {
fn as_ref(&self) -> &str {
self.as_str()
}
}
impl From<FileSystemPathBuf> for VfsPath {
fn from(value: FileSystemPathBuf) -> Self {
Self::FileSystem(value)
}
}
impl From<&FileSystemPath> for VfsPath {
fn from(value: &FileSystemPath) -> Self {
VfsPath::FileSystem(value.to_path_buf())
}
}
impl From<VendoredPathBuf> for VfsPath {
fn from(value: VendoredPathBuf) -> Self {
Self::Vendored(value)
}
}
impl From<&VendoredPath> for VfsPath {
fn from(value: &VendoredPath) -> Self {
Self::Vendored(value.to_path_buf())
}
}
impl PartialEq<FileSystemPath> for VfsPath {
#[inline]
fn eq(&self, other: &FileSystemPath) -> bool {
self.as_file_system_path()
.is_some_and(|self_path| self_path == other)
}
}
impl PartialEq<VfsPath> for FileSystemPath {
#[inline]
fn eq(&self, other: &VfsPath) -> bool {
other == self
}
}
impl PartialEq<FileSystemPathBuf> for VfsPath {
#[inline]
fn eq(&self, other: &FileSystemPathBuf) -> bool {
self == other.as_path()
}
}
impl PartialEq<VfsPath> for FileSystemPathBuf {
fn eq(&self, other: &VfsPath) -> bool {
other == self
}
}
impl PartialEq<VendoredPath> for VfsPath {
#[inline]
fn eq(&self, other: &VendoredPath) -> bool {
self.as_vendored_path()
.is_some_and(|self_path| self_path == other)
}
}
impl PartialEq<VfsPath> for VendoredPath {
#[inline]
fn eq(&self, other: &VfsPath) -> bool {
other == self
}
}
impl PartialEq<VendoredPathBuf> for VfsPath {
#[inline]
fn eq(&self, other: &VendoredPathBuf) -> bool {
other.as_path() == self
}
}
impl PartialEq<VfsPath> for VendoredPathBuf {
#[inline]
fn eq(&self, other: &VfsPath) -> bool {
other == self
}
}