language-servers/tinymist
1
0
Fork 0
mirror of https://github.com/Myriad-Dreamin/tinymist.git synced 2025-08-03 17:58:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

feat: encode and use workspace information into PackageSpec (#1187)

Browse source 
* feat: remove an unused API

* feat: encode workspace information into `PackageSpec`

* feat: remove unused real_path

* feat: remove unused mtime

* feat: add ResolveAccessModel

* feat: implement id overlay semantics

* feat: remove mtime checking in overlay model

* feat: remove mtime checking in notify model

* feat: format ids

* fix: cases

* feat: resolve root by world

* dev: add untitled root

* fix: warnings

* fix: a wrong usage

* fix: snapshots

* fix: tests
This commit is contained in:
Myriad-Dreamin 2025-01-19 11:51:00 +08:00 • committed by GitHub
parent a25d208124
commit 56714675b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 835 additions and 774 deletions
Download patch file Download diff file Expand all files Collapse all files

1
crates/tinymist-vfs/Cargo.toml
View file

@ -15,6 +15,7 @@ tinymist-std = { workspace = true, features = ["typst"] }
parking_lot.workspace = true
nohash-hasher.workspace = true
indexmap.workspace = true
comemo.workspace = true
log.workspace = true
rpds = "1"

36
crates/tinymist-vfs/src/browser.rs
View file

@ -1,10 +1,9 @@
use std::path::Path;
use tinymist_std::ImmutPath;
use typst::diag::{FileError, FileResult};
use wasm_bindgen::prelude::*;
use crate::{AccessModel, Bytes, Time};
use crate::{Bytes, PathAccessModel};
/// Provides proxy access model from typst compiler to some JavaScript
/// implementation.
@ -23,24 +22,7 @@ pub struct ProxyAccessModel {
pub read_all_fn: js_sys::Function,
}
impl AccessModel for ProxyAccessModel {
fn mtime(&self, src: &Path) -> FileResult<Time> {
self.mtime_fn
.call1(&self.context, &src.to_string_lossy().as_ref().into())
.map(|v| {
let v = v.as_f64().unwrap();
Time::UNIX_EPOCH + std::time::Duration::from_secs_f64(v)
})
.map_err(|e| {
web_sys::console::error_3(
&"typst_ts::compiler::ProxyAccessModel::mtime failure".into(),
&src.to_string_lossy().as_ref().into(),
&e,
);
FileError::AccessDenied
})
}
impl PathAccessModel for ProxyAccessModel {
fn is_file(&self, src: &Path) -> FileResult<bool> {
self.is_file_fn
.call1(&self.context, &src.to_string_lossy().as_ref().into())
@ -55,20 +37,6 @@ impl AccessModel for ProxyAccessModel {
})
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
self.real_path_fn
.call1(&self.context, &src.to_string_lossy().as_ref().into())
.map(|v| Path::new(&v.as_string().unwrap()).into())
.map_err(|e| {
web_sys::console::error_3(
&"typst_ts::compiler::ProxyAccessModel::real_path failure".into(),
&src.to_string_lossy().as_ref().into(),
&e,
);
FileError::AccessDenied
})
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
let data = self
.read_all_fn

20
crates/tinymist-vfs/src/dummy.rs
View file

@ -1,10 +1,8 @@
use std::path::Path;
use tinymist_std::ImmutPath;
use typst::diag::{FileError, FileResult};
use super::AccessModel;
use crate::{Bytes, Time};
use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
/// Provides dummy access model.
///
@ -15,16 +13,18 @@ use crate::{Bytes, Time};
pub struct DummyAccessModel;
impl AccessModel for DummyAccessModel {
fn mtime(&self, _src: &Path) -> FileResult<Time> {
Ok(Time::UNIX_EPOCH)
}
fn is_file(&self, _src: &Path) -> FileResult<bool> {
fn is_file(&self, _src: TypstFileId) -> FileResult<bool> {
Ok(true)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
Ok(src.into())
fn content(&self, _src: TypstFileId) -> FileResult<Bytes> {
Err(FileError::AccessDenied)
}
}
impl PathAccessModel for DummyAccessModel {
fn is_file(&self, _src: &Path) -> FileResult<bool> {
Ok(true)
}
fn content(&self, _src: &Path) -> FileResult<Bytes> {

202
crates/tinymist-vfs/src/lib.rs
View file

@ -25,40 +25,51 @@ pub mod notify;
/// Provides overlay access model which allows to shadow the underlying access
/// model with memory contents.
pub mod overlay;
/// Provides resolve access model.
pub mod resolve;
/// Provides trace access model which traces the underlying access model.
pub mod trace;
mod utils;
mod path_interner;
mod path_mapper;
use notify::{FilesystemEvent, NotifyAccessModel};
pub use path_mapper::{PathResolution, RootResolver, WorkspaceResolution, WorkspaceResolver};
use resolve::ResolveAccessModel;
pub use typst::foundations::Bytes;
pub use typst::syntax::FileId as TypstFileId;
pub use tinymist_std::time::Time;
pub use tinymist_std::ImmutPath;
pub(crate) use path_interner::PathInterner;
use core::fmt;
use std::{collections::HashMap, hash::Hash, path::Path, sync::Arc};
use std::{path::Path, sync::Arc};
use parking_lot::{Mutex, RwLock};
use tinymist_std::path::PathClean;
use parking_lot::RwLock;
use typst::diag::{FileError, FileResult};
use self::{
notify::{FilesystemEvent, NotifyAccessModel},
overlay::OverlayAccessModel,
};
use self::overlay::OverlayAccessModel;
/// Handle to a file in [`Vfs`]
///
/// Most functions in typst-ts use this when they need to refer to a file.
#[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)]
pub struct FileId(pub u32);
pub type FileId = TypstFileId;
/// safe because `FileId` is a new type of `u32`
impl nohash_hasher::IsEnabled for FileId {}
/// A trait for accessing underlying file system.
///
/// This trait is simplified by [`Vfs`] and requires a minimal method set for
/// typst compilation.
pub trait PathAccessModel {
/// Clear the cache of the access model.
///
/// This is called when the vfs is reset. See [`Vfs`]'s reset method for
/// more information.
fn clear(&mut self) {}
/// Return whether a path is corresponding to a file.
fn is_file(&self, src: &Path) -> FileResult<bool>;
/// Return the content of a file entry.
fn content(&self, src: &Path) -> FileResult<Bytes>;
}
/// A trait for accessing underlying file system.
///
@ -70,23 +81,12 @@ pub trait AccessModel {
/// This is called when the vfs is reset. See [`Vfs`]'s reset method for
/// more information.
fn clear(&mut self) {}
/// Return a mtime corresponding to the path.
///
/// Note: vfs won't touch the file entry if mtime is same between vfs reset
/// lifecycles for performance design.
fn mtime(&self, src: &Path) -> FileResult<Time>;
/// Return whether a path is corresponding to a file.
fn is_file(&self, src: &Path) -> FileResult<bool>;
/// Return the real path before creating a vfs file entry.
///
/// Note: vfs will fetch the file entry once if multiple paths shares a same
/// real path.
fn real_path(&self, src: &Path) -> FileResult<ImmutPath>;
fn is_file(&self, src: TypstFileId) -> FileResult<bool>;
/// Return the content of a file entry.
fn content(&self, src: &Path) -> FileResult<Bytes>;
fn content(&self, src: TypstFileId) -> FileResult<Bytes>;
}
#[derive(Clone)]
@ -102,110 +102,60 @@ impl<M> SharedAccessModel<M> {
}
}
impl<M> AccessModel for SharedAccessModel<M>
impl<M> PathAccessModel for SharedAccessModel<M>
where
M: AccessModel,
M: PathAccessModel,
{
fn clear(&mut self) {
self.inner.write().clear();
}
fn mtime(&self, src: &Path) -> FileResult<Time> {
self.inner.read().mtime(src)
}
fn is_file(&self, src: &Path) -> FileResult<bool> {
self.inner.read().is_file(src)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
self.inner.read().real_path(src)
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
self.inner.read().content(src)
}
}
type VfsPathAccessModel<M> = OverlayAccessModel<ImmutPath, NotifyAccessModel<SharedAccessModel<M>>>;
/// we add notify access model here since notify access model doesn't introduce
/// overheads by our observation
type VfsAccessModel<M> = OverlayAccessModel<NotifyAccessModel<SharedAccessModel<M>>>;
type VfsAccessModel<M> = OverlayAccessModel<TypstFileId, ResolveAccessModel<VfsPathAccessModel<M>>>;
pub trait FsProvider {
/// Arbitrary one of file path corresponding to the given `id`.
fn file_path(&self, id: FileId) -> ImmutPath;
fn file_path(&self, id: TypstFileId) -> FileResult<PathResolution>;
fn mtime(&self, id: FileId) -> FileResult<Time>;
fn read(&self, id: TypstFileId) -> FileResult<Bytes>;
fn read(&self, id: FileId) -> FileResult<Bytes>;
fn is_file(&self, id: FileId) -> FileResult<bool>;
fn is_file(&self, id: TypstFileId) -> FileResult<bool>;
}
#[derive(Default)]
struct PathMapper {
/// Map from path to slot index.
///
/// Note: we use a owned [`FileId`] here, which is resultant from
/// [`PathInterner`]
id_cache: RwLock<HashMap<ImmutPath, FileId>>,
/// The path interner for canonical paths.
intern: Mutex<PathInterner<ImmutPath, ()>>,
}
impl PathMapper {
/// Id of the given path if it exists in the `Vfs` and is not deleted.
pub fn file_id(&self, path: &Path) -> FileId {
let quick_id = self.id_cache.read().get(path).copied();
if let Some(id) = quick_id {
return id;
}
let path: ImmutPath = path.clean().as_path().into();
let mut path_interner = self.intern.lock();
let id = path_interner.intern(path.clone(), ()).0;
let mut path2slot = self.id_cache.write();
path2slot.insert(path.clone(), id);
id
}
/// File path corresponding to the given `file_id`.
pub fn file_path(&self, file_id: FileId) -> ImmutPath {
let path_interner = self.intern.lock();
path_interner.lookup(file_id).clone()
}
}
/// Create a new `Vfs` harnessing over the given `access_model` specific for
/// `reflexo_world::CompilerWorld`. With vfs, we can minimize the
/// implementation overhead for [`AccessModel`] trait.
pub struct Vfs<M: AccessModel + Sized> {
paths: Arc<PathMapper>,
pub struct Vfs<M: PathAccessModel + Sized> {
// access_model: TraceAccessModel<VfsAccessModel<M>>,
/// The wrapped access model.
access_model: VfsAccessModel<M>,
}
impl<M: AccessModel + Sized> fmt::Debug for Vfs<M> {
impl<M: PathAccessModel + Sized> fmt::Debug for Vfs<M> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Vfs").finish()
}
}
impl<M: AccessModel + Clone + Sized> Vfs<M> {
impl<M: PathAccessModel + Clone + Sized> Vfs<M> {
pub fn snapshot(&self) -> Self {
Self {
paths: self.paths.clone(),
access_model: self.access_model.clone(),
}
}
}
impl<M: AccessModel + Sized> Vfs<M> {
impl<M: PathAccessModel + Sized> Vfs<M> {
/// Create a new `Vfs` with a given `access_model`.
///
/// Retrieving an [`AccessModel`], it will further wrap the access model
@ -218,18 +168,20 @@ impl<M: AccessModel + Sized> Vfs<M> {
/// the vfs is watching the file system.
///
/// See [`AccessModel`] for more information.
pub fn new(access_model: M) -> Self {
pub fn new(resolver: Arc<dyn RootResolver + Send + Sync>, access_model: M) -> Self {
let access_model = SharedAccessModel::new(access_model);
let access_model = NotifyAccessModel::new(access_model);
let access_model = OverlayAccessModel::new(access_model);
let access_model = ResolveAccessModel {
resolver,
inner: access_model,
};
let access_model = OverlayAccessModel::new(access_model);
// If you want to trace the access model, uncomment the following line
// let access_model = TraceAccessModel::new(access_model);
Self {
paths: Default::default(),
access_model,
}
Self { access_model }
}
/// Reset the source file and path references.
@ -243,35 +195,61 @@ impl<M: AccessModel + Sized> Vfs<M> {
self.access_model.clear();
}
/// Resolve the real path for a file id.
pub fn file_path(&self, id: TypstFileId) -> Result<PathResolution, FileError> {
self.access_model.inner.resolver.path_for_id(id)
}
/// Reset the shadowing files in [`OverlayAccessModel`].
///
/// Note: This function is independent from [`Vfs::reset`].
pub fn reset_shadow(&mut self) {
self.access_model.clear_shadow();
self.access_model.inner.inner.clear_shadow();
}
/// Get paths to all the shadowing files in [`OverlayAccessModel`].
pub fn shadow_paths(&self) -> Vec<Arc<Path>> {
pub fn shadow_paths(&self) -> Vec<ImmutPath> {
self.access_model.inner.inner.file_paths()
}
/// Get paths to all the shadowing files in [`OverlayAccessModel`].
pub fn shadow_ids(&self) -> Vec<TypstFileId> {
self.access_model.file_paths()
}
/// Add a shadowing file to the [`OverlayAccessModel`].
pub fn map_shadow(&mut self, path: &Path, content: Bytes) -> FileResult<()> {
self.access_model.add_file(path.into(), content);
self.access_model
.inner
.inner
.add_file(path, content, |c| c.into());
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`].
pub fn remove_shadow(&mut self, path: &Path) {
self.access_model.remove_file(path);
self.access_model.inner.inner.remove_file(path);
}
/// Add a shadowing file to the [`OverlayAccessModel`] by file id.
pub fn map_shadow_by_id(&mut self, file_id: TypstFileId, content: Bytes) -> FileResult<()> {
self.access_model.add_file(&file_id, content, |c| *c);
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`] by file id.
pub fn remove_shadow_by_id(&mut self, file_id: TypstFileId) {
self.access_model.remove_file(&file_id);
}
/// Let the vfs notify the access model with a filesystem event.
///
/// See [`NotifyAccessModel`] for more information.
pub fn notify_fs_event(&mut self, event: FilesystemEvent) {
self.access_model.inner.notify(event);
self.access_model.inner.inner.inner.notify(event);
}
/// Returns the overall memory usage for the stored files.
@ -279,36 +257,18 @@ impl<M: AccessModel + Sized> Vfs<M> {
0
}
/// Id of the given path if it exists in the `Vfs` and is not deleted.
pub fn file_id(&self, path: &Path) -> FileId {
self.paths.file_id(path)
}
/// Read a file.
pub fn read(&self, path: &Path) -> FileResult<Bytes> {
pub fn read(&self, path: TypstFileId) -> FileResult<Bytes> {
if self.access_model.is_file(path)? {
self.access_model.content(path)
} else {
Err(FileError::IsDirectory)
}
}
}
impl<M: AccessModel> FsProvider for Vfs<M> {
fn file_path(&self, id: FileId) -> ImmutPath {
self.paths.file_path(id)
}
fn mtime(&self, src: FileId) -> FileResult<Time> {
self.access_model.mtime(&self.file_path(src))
}
fn read(&self, src: FileId) -> FileResult<Bytes> {
self.access_model.content(&self.file_path(src))
}
fn is_file(&self, src: FileId) -> FileResult<bool> {
self.access_model.is_file(&self.file_path(src))
/// Whether the given path is a file.
pub fn is_file(&self, path: TypstFileId) -> FileResult<bool> {
self.access_model.is_file(path)
}
}

67
crates/tinymist-vfs/src/notify.rs
View file

@ -4,33 +4,20 @@ use std::path::Path;
use rpds::RedBlackTreeMapSync;
use typst::diag::{FileError, FileResult};
use crate::{AccessModel, Bytes, ImmutPath};
/// internal representation of [`NotifyFile`]
#[derive(Debug, Clone)]
struct NotifyFileRepr {
mtime: crate::Time,
content: Bytes,
}
use crate::{Bytes, ImmutPath, PathAccessModel};
/// A file snapshot that is notified by some external source
///
/// Note: The error is boxed to avoid large stack size
#[derive(Clone)]
pub struct FileSnapshot(Result<NotifyFileRepr, Box<FileError>>);
pub struct FileSnapshot(Result<Bytes, Box<FileError>>);
impl fmt::Debug for FileSnapshot {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.0.as_ref() {
Ok(v) => f
.debug_struct("FileSnapshot")
.field("mtime", &v.mtime)
.field(
"content",
&FileContent {
len: v.content.len(),
},
)
.field("content", &FileContent { len: v.len() })
.finish(),
Err(e) => f.debug_struct("FileSnapshot").field("error", &e).finish(),
}
@ -38,37 +25,25 @@ impl fmt::Debug for FileSnapshot {
}
impl FileSnapshot {
/// Access the internal data of the file snapshot
/// content of the file
#[inline]
#[track_caller]
fn retrieve<'a, T>(&'a self, f: impl FnOnce(&'a NotifyFileRepr) -> T) -> FileResult<T> {
self.0.as_ref().map(f).map_err(|e| *e.clone())
}
/// mtime of the file
pub fn mtime(&self) -> FileResult<&crate::Time> {
self.retrieve(|e| &e.mtime)
}
/// content of the file
pub fn content(&self) -> FileResult<&Bytes> {
self.retrieve(|e| &e.content)
self.0.as_ref().map_err(|e| *e.clone())
}
/// Whether the related file is a file
#[inline]
#[track_caller]
pub fn is_file(&self) -> FileResult<bool> {
self.retrieve(|_| true)
self.content().map(|_| true)
}
}
/// Convenient function to create a [`FileSnapshot`] from tuple
impl From<FileResult<(crate::Time, Bytes)>> for FileSnapshot {
fn from(result: FileResult<(crate::Time, Bytes)>) -> Self {
Self(
result
.map(|(mtime, content)| NotifyFileRepr { mtime, content })
.map_err(Box::new),
)
impl From<FileResult<Bytes>> for FileSnapshot {
fn from(result: FileResult<Bytes>) -> Self {
Self(result.map_err(Box::new))
}
}
@ -212,7 +187,7 @@ pub struct NotifyAccessModel<M> {
pub inner: M,
}
impl<M: AccessModel> NotifyAccessModel<M> {
impl<M: PathAccessModel> NotifyAccessModel<M> {
/// Create a new notify access model
pub fn new(inner: M) -> Self {
Self {
@ -238,15 +213,7 @@ impl<M: AccessModel> NotifyAccessModel<M> {
}
}
impl<M: AccessModel> AccessModel for NotifyAccessModel<M> {
fn mtime(&self, src: &Path) -> FileResult<crate::Time> {
if let Some(entry) = self.files.get(src) {
return entry.mtime().cloned();
}
self.inner.mtime(src)
}
impl<M: PathAccessModel> PathAccessModel for NotifyAccessModel<M> {
fn is_file(&self, src: &Path) -> FileResult<bool> {
if let Some(entry) = self.files.get(src) {
return entry.is_file();
@ -255,14 +222,6 @@ impl<M: AccessModel> AccessModel for NotifyAccessModel<M> {
self.inner.is_file(src)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
if self.files.contains_key(src) {
return Ok(src.into());
}
self.inner.real_path(src)
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
if let Some(entry) = self.files.get(src) {
return entry.content().cloned();

92
crates/tinymist-vfs/src/overlay.rs
View file

@ -1,28 +1,21 @@
use std::path::Path;
use std::sync::Arc;
use std::{borrow::Borrow, cmp::Ord, path::Path};
use rpds::RedBlackTreeMapSync;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{AccessModel, Bytes, Time};
#[derive(Debug, Clone)]
struct OverlayFileMeta {
mt: Time,
content: Bytes,
}
use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
/// Provides overlay access model which allows to shadow the underlying access
/// model with memory contents.
#[derive(Default, Debug, Clone)]
pub struct OverlayAccessModel<M> {
files: RedBlackTreeMapSync<Arc<Path>, OverlayFileMeta>,
pub struct OverlayAccessModel<K: Ord, M> {
files: RedBlackTreeMapSync<K, Bytes>,
/// The underlying access model
pub inner: M,
}
impl<M: AccessModel> OverlayAccessModel<M> {
impl<K: Ord + Clone, M> OverlayAccessModel<K, M> {
/// Create a new [`OverlayAccessModel`] with the given inner access model
pub fn new(inner: M) -> Self {
Self {
@ -47,54 +40,35 @@ impl<M: AccessModel> OverlayAccessModel<M> {
}
/// Get the shadowed file paths
pub fn file_paths(&self) -> Vec<Arc<Path>> {
pub fn file_paths(&self) -> Vec<K> {
self.files.keys().cloned().collect()
}
/// Add a shadow file to the [`OverlayAccessModel`]
pub fn add_file(&mut self, path: Arc<Path>, content: Bytes) {
// we change mt every time, since content almost changes every time
// Note: we can still benefit from cache, since we incrementally parse source
let mt = tinymist_std::time::now();
let meta = OverlayFileMeta { mt, content };
match self.files.get_mut(&path) {
pub fn add_file<Q: Ord + ?Sized>(&mut self, path: &Q, content: Bytes, cast: impl Fn(&Q) -> K)
where
K: Borrow<Q>,
{
match self.files.get_mut(path) {
Some(e) => {
if e.mt == meta.mt && e.content != meta.content {
e.mt = meta
.mt
// [`crate::Time`] has a minimum resolution of 1ms
// we negate the time by 1ms so that the time is always
// invalidated
.checked_sub(std::time::Duration::from_millis(1))
.unwrap();
e.content = meta.content.clone();
} else {
*e = meta.clone();
}
*e = content;
}
None => {
self.files.insert_mut(path, meta);
self.files.insert_mut(cast(path), content);
}
}
}
/// Remove a shadow file from the [`OverlayAccessModel`]
pub fn remove_file(&mut self, path: &Path) {
pub fn remove_file<Q: Ord + ?Sized>(&mut self, path: &Q)
where
K: Borrow<Q>,
{
self.files.remove_mut(path);
}
}
impl<M: AccessModel> AccessModel for OverlayAccessModel<M> {
fn mtime(&self, src: &Path) -> FileResult<Time> {
if let Some(meta) = self.files.get(src) {
return Ok(meta.mt);
}
self.inner.mtime(src)
}
impl<M: PathAccessModel> PathAccessModel for OverlayAccessModel<ImmutPath, M> {
fn is_file(&self, src: &Path) -> FileResult<bool> {
if self.files.get(src).is_some() {
return Ok(true);
@ -103,17 +77,27 @@ impl<M: AccessModel> AccessModel for OverlayAccessModel<M> {
self.inner.is_file(src)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
if self.files.get(src).is_some() {
return Ok(src.into());
}
self.inner.real_path(src)
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
if let Some(meta) = self.files.get(src) {
return Ok(meta.content.clone());
if let Some(content) = self.files.get(src) {
return Ok(content.clone());
}
self.inner.content(src)
}
}
impl<M: AccessModel> AccessModel for OverlayAccessModel<TypstFileId, M> {
fn is_file(&self, src: TypstFileId) -> FileResult<bool> {
if self.files.get(&src).is_some() {
return Ok(true);
}
self.inner.is_file(src)
}
fn content(&self, src: TypstFileId) -> FileResult<Bytes> {
if let Some(content) = self.files.get(&src) {
return Ok(content.clone());
}
self.inner.content(src)

71
crates/tinymist-vfs/src/path_interner.rs
View file

@ -1,71 +0,0 @@
//! Maps paths to compact integer ids. We don't care about clearings paths which
//! no longer exist -- the assumption is total size of paths we ever look at is
//! not too big.
use std::hash::{BuildHasherDefault, Hash};
use indexmap::IndexMap;
use tinymist_std::hash::FxHasher;
use super::FileId;
/// Structure to map between [`VfsPath`] and [`FileId`].
#[derive(Debug)]
pub(crate) struct PathInterner<P, Ext = ()> {
map: IndexMap<P, Ext, BuildHasherDefault<FxHasher>>,
}
impl<P, Ext> Default for PathInterner<P, Ext> {
fn default() -> Self {
Self {
map: IndexMap::default(),
}
}
}
impl<P: Hash + Eq, Ext> PathInterner<P, Ext> {
/// Scan through each value in the set and keep those where the
/// closure `keep` returns `true`.
///
/// The elements are visited in order, and remaining elements keep their
/// order.
///
/// Computes in **O(n)** time (average).
#[allow(dead_code)]
pub fn retain(&mut self, keep: impl FnMut(&P, &mut Ext) -> bool) {
self.map.retain(keep)
}
/// Insert `path` in `self`.
///
/// - If `path` already exists in `self`, returns its associated id;
/// - Else, returns a newly allocated id.
#[inline]
pub(crate) fn intern(&mut self, path: P, ext: Ext) -> (FileId, Option<&mut Ext>) {
let (id, _) = self.map.insert_full(path, ext);
assert!(id < u32::MAX as usize);
(FileId(id as u32), None)
}
/// Returns the path corresponding to `id`.
///
/// # Panics
///
/// Panics if `id` does not exists in `self`.
#[allow(dead_code)]
pub(crate) fn lookup(&self, id: FileId) -> &P {
self.map.get_index(id.0 as usize).unwrap().0
}
}
#[cfg(test)]
mod tests {
use super::PathInterner;
use std::path::PathBuf;
#[test]
fn test_interner_path_buf() {
let mut interner = PathInterner::<PathBuf>::default();
let (id, ..) = interner.intern(PathBuf::from("foo"), ());
assert_eq!(interner.lookup(id), &PathBuf::from("foo"));
}
}

286
crates/tinymist-vfs/src/path_mapper.rs Normal file
View file

@ -0,0 +1,286 @@
//! Maps paths to compact integer ids. We don't care about clearings paths which
//! no longer exist -- the assumption is total size of paths we ever look at is
//! not too big.
use core::fmt;
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use parking_lot::RwLock;
use tinymist_std::path::PathClean;
use tinymist_std::ImmutPath;
use typst::diag::{eco_format, EcoString, FileError, FileResult};
use typst::syntax::package::{PackageSpec, PackageVersion};
use typst::syntax::VirtualPath;
use super::TypstFileId;
pub enum PathResolution {
Resolved(PathBuf),
Rootless(Cow<'static, VirtualPath>),
}
impl PathResolution {
pub fn to_err(self) -> FileResult<PathBuf> {
match self {
PathResolution::Resolved(path) => Ok(path),
PathResolution::Rootless(_) => Err(FileError::AccessDenied),
}
}
pub fn as_path(&self) -> &Path {
match self {
PathResolution::Resolved(path) => path.as_path(),
PathResolution::Rootless(path) => path.as_rooted_path(),
}
}
pub fn join(&self, path: &str) -> FileResult<PathResolution> {
match self {
PathResolution::Resolved(path) => Ok(PathResolution::Resolved(path.join(path))),
PathResolution::Rootless(root) => {
Ok(PathResolution::Rootless(Cow::Owned(root.join(path))))
}
}
}
}
pub trait RootResolver {
fn path_for_id(&self, file_id: TypstFileId) -> FileResult<PathResolution> {
use WorkspaceResolution::*;
let root = match WorkspaceResolver::resolve(file_id)? {
Workspace(id) => id.path().clone(),
Package => {
self.resolve_package_root(file_id.package().expect("not a file in package"))?
}
UntitledRooted(..) | Rootless => {
return Ok(PathResolution::Rootless(Cow::Borrowed(file_id.vpath())))
}
};
file_id
.vpath()
.resolve(&root)
.map(PathResolution::Resolved)
.ok_or_else(|| FileError::AccessDenied)
}
fn resolve_root(&self, file_id: TypstFileId) -> FileResult<Option<ImmutPath>> {
use WorkspaceResolution::*;
match WorkspaceResolver::resolve(file_id)? {
Workspace(id) | UntitledRooted(id) => Ok(Some(id.path().clone())),
Rootless => Ok(None),
Package => self
.resolve_package_root(file_id.package().expect("not a file in package"))
.map(Some),
}
}
fn resolve_package_root(&self, pkg: &PackageSpec) -> FileResult<ImmutPath>;
}
#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct WorkspaceId(u16);
const NO_VERSION: PackageVersion = PackageVersion {
major: 0,
minor: 0,
patch: 0,
};
const UNTITLED_ROOT: PackageVersion = PackageVersion {
major: 0,
minor: 0,
patch: 1,
};
impl WorkspaceId {
fn package(&self) -> PackageSpec {
PackageSpec {
namespace: WorkspaceResolver::WORKSPACE_NS.clone(),
name: eco_format!("p{}", self.0),
version: NO_VERSION,
}
}
fn untitled_root(&self) -> PackageSpec {
PackageSpec {
namespace: WorkspaceResolver::WORKSPACE_NS.clone(),
name: eco_format!("p{}", self.0),
version: UNTITLED_ROOT,
}
}
pub fn path(&self) -> ImmutPath {
let interner = INTERNER.read();
interner
.from_id
.get(self.0 as usize)
.expect("invalid workspace id")
.clone()
}
fn from_package_name(name: &str) -> Option<WorkspaceId> {
if !name.starts_with("p") {
return None;
}
let num = name[1..].parse().ok()?;
Some(WorkspaceId(num))
}
}
/// The global package-path interner.
static INTERNER: LazyLock<RwLock<Interner>> = LazyLock::new(|| {
RwLock::new(Interner {
to_id: HashMap::new(),
from_id: Vec::new(),
})
});
pub enum WorkspaceResolution {
Workspace(WorkspaceId),
UntitledRooted(WorkspaceId),
Rootless,
Package,
}
/// A package-path interner.
struct Interner {
to_id: HashMap<ImmutPath, WorkspaceId>,
from_id: Vec<ImmutPath>,
}
#[derive(Default)]
pub struct WorkspaceResolver {}
impl WorkspaceResolver {
pub const WORKSPACE_NS: EcoString = EcoString::inline("ws");
pub fn is_workspace_file(fid: TypstFileId) -> bool {
fid.package()
.is_some_and(|p| p.namespace == WorkspaceResolver::WORKSPACE_NS)
}
pub fn is_package_file(fid: TypstFileId) -> bool {
fid.package()
.is_some_and(|p| p.namespace != WorkspaceResolver::WORKSPACE_NS)
}
/// Id of the given path if it exists in the `Vfs` and is not deleted.
pub fn workspace_id(root: &ImmutPath) -> WorkspaceId {
// Try to find an existing entry that we can reuse.
//
// We could check with just a read lock, but if the pair is not yet
// present, we would then need to recheck after acquiring a write lock,
// which is probably not worth it.
let mut interner = INTERNER.write();
if let Some(&id) = interner.to_id.get(root) {
return id;
}
let root = ImmutPath::from(root.clean());
// Create a new entry forever by leaking the pair. We can't leak more
// than 2^16 pair (and typically will leak a lot less), so its not a
// big deal.
let num = interner.from_id.len().try_into().expect("out of file ids");
let id = WorkspaceId(num);
interner.to_id.insert(root.clone(), id);
interner.from_id.push(root.clone());
id
}
/// Creates a file id for a rootless file.
pub fn rootless_file(path: VirtualPath) -> TypstFileId {
TypstFileId::new(None, path)
}
/// Creates a file id for a rootless file.
pub fn file_with_parent_root(path: &Path) -> Option<TypstFileId> {
if !path.is_absolute() {
return None;
}
let parent = path.parent()?;
let parent = ImmutPath::from(parent);
let path = VirtualPath::new(path.file_name()?);
Some(Self::workspace_file(Some(&parent), path))
}
/// Creates a file id for a file in some workspace. The `root` is the root
/// directory of the workspace. If `root` is `None`, the source code at the
/// `path` will not be able to access physical files.
pub fn workspace_file(root: Option<&ImmutPath>, path: VirtualPath) -> TypstFileId {
let workspace = root.map(Self::workspace_id);
TypstFileId::new(workspace.as_ref().map(WorkspaceId::package), path)
}
/// Mounts an untiled file to some workspace. The `root` is the
/// root directory of the workspace. If `root` is `None`, the source
/// code at the `path` will not be able to access physical files.
pub fn rooted_untitled(root: Option<&ImmutPath>, path: VirtualPath) -> TypstFileId {
let workspace = root.map(Self::workspace_id);
TypstFileId::new(workspace.as_ref().map(WorkspaceId::untitled_root), path)
}
/// File path corresponding to the given `fid`.
pub fn resolve(fid: TypstFileId) -> FileResult<WorkspaceResolution> {
let Some(package) = fid.package() else {
return Ok(WorkspaceResolution::Rootless);
};
match package.namespace.as_str() {
"ws" => {
let id = WorkspaceId::from_package_name(&package.name).ok_or_else(|| {
FileError::Other(Some(eco_format!("bad workspace id: {fid:?}")))
})?;
Ok(if package.version == UNTITLED_ROOT {
WorkspaceResolution::UntitledRooted(id)
} else {
WorkspaceResolution::Workspace(id)
})
}
_ => Ok(WorkspaceResolution::Package),
}
}
/// File path corresponding to the given `fid`.
pub fn display(id: Option<TypstFileId>) -> Resolving {
Resolving { id }
}
}
pub struct Resolving {
id: Option<TypstFileId>,
}
impl fmt::Debug for Resolving {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use WorkspaceResolution::*;
let Some(id) = self.id else {
return write!(f, "None");
};
let path = match WorkspaceResolver::resolve(id) {
Ok(Workspace(workspace)) => id.vpath().resolve(&workspace.path()),
Ok(UntitledRooted(..)) => Some(id.vpath().as_rootless_path().to_owned()),
Ok(Rootless | Package) | Err(_) => None,
};
if let Some(path) = path {
write!(f, "{}", path.display())
} else {
write!(f, "{:?}", self.id)
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn test_interner_untitled() {}
}

30
crates/tinymist-vfs/src/resolve.rs Normal file
View file

@ -0,0 +1,30 @@
use std::{fmt::Debug, sync::Arc};
use typst::diag::FileResult;
use crate::{path_mapper::RootResolver, AccessModel, Bytes, PathAccessModel, TypstFileId};
/// Provides resolve access model.
#[derive(Clone)]
pub struct ResolveAccessModel<M> {
pub resolver: Arc<dyn RootResolver + Send + Sync>,
pub inner: M,
}
impl<M> Debug for ResolveAccessModel<M> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ResolveAccessModel").finish()
}
}
impl<M: PathAccessModel> AccessModel for ResolveAccessModel<M> {
fn is_file(&self, fid: TypstFileId) -> FileResult<bool> {
self.inner
.is_file(&self.resolver.path_for_id(fid)?.to_err()?)
}
fn content(&self, fid: TypstFileId) -> FileResult<Bytes> {
self.inner
.content(&self.resolver.path_for_id(fid)?.to_err()?)
}
}

17
crates/tinymist-vfs/src/system.rs
View file

@ -2,8 +2,8 @@ use std::{fs::File, io::Read, path::Path};
use typst::diag::{FileError, FileResult};
use crate::{AccessModel, Bytes, Time};
use tinymist_std::{ImmutPath, ReadAllOnce};
use crate::{Bytes, PathAccessModel};
use tinymist_std::ReadAllOnce;
/// Provides SystemAccessModel that makes access to the local file system for
/// system compilation.
@ -14,27 +14,17 @@ impl SystemAccessModel {
fn stat(&self, src: &Path) -> std::io::Result<SystemFileMeta> {
let meta = std::fs::metadata(src)?;
Ok(SystemFileMeta {
mt: meta.modified()?,
is_file: meta.is_file(),
})
}
}
impl AccessModel for SystemAccessModel {
fn mtime(&self, src: &Path) -> FileResult<Time> {
let f = |e| FileError::from_io(e, src);
Ok(self.stat(src).map_err(f)?.mt)
}
impl PathAccessModel for SystemAccessModel {
fn is_file(&self, src: &Path) -> FileResult<bool> {
let f = |e| FileError::from_io(e, src);
Ok(self.stat(src).map_err(f)?.is_file)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
Ok(src.into())
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
let f = |e| FileError::from_io(e, src);
let mut buf = Vec::<u8>::new();
@ -78,6 +68,5 @@ impl ReadAllOnce for LazyFile {
/// Meta data of a file in the local file system.
#[derive(Debug, Clone, Copy)]
pub struct SystemFileMeta {
mt: std::time::SystemTime,
is_file: bool,
}

34
crates/tinymist-vfs/src/trace.rs
View file

@ -1,9 +1,8 @@
use std::{path::Path, sync::atomic::AtomicU64};
use std::sync::atomic::AtomicU64;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{AccessModel, Bytes};
use crate::{AccessModel, Bytes, TypstFileId};
/// Provides trace access model which traces the underlying access model.
///
@ -30,20 +29,7 @@ impl<M: AccessModel + Sized> AccessModel for TraceAccessModel<M> {
self.inner.clear();
}
fn mtime(&self, src: &Path) -> FileResult<crate::Time> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.mtime(src);
let elapsed = instant.elapsed();
// self.trace[0] += elapsed.as_nanos() as u64;
self.trace[0].fetch_add(
elapsed.as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
crate::utils::console_log!("mtime: {:?} {:?} => {:?}", src, elapsed, res);
res
}
fn is_file(&self, src: &Path) -> FileResult<bool> {
fn is_file(&self, src: TypstFileId) -> FileResult<bool> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.is_file(src);
let elapsed = instant.elapsed();
@ -55,19 +41,7 @@ impl<M: AccessModel + Sized> AccessModel for TraceAccessModel<M> {
res
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.real_path(src);
let elapsed = instant.elapsed();
self.trace[2].fetch_add(
elapsed.as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
crate::utils::console_log!("real_path: {:?} {:?}", src, elapsed);
res
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
fn content(&self, src: TypstFileId) -> FileResult<Bytes> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.content(src);
let elapsed = instant.elapsed();