language-servers/tinymist
1
0
Fork 0
mirror of https://github.com/Myriad-Dreamin/tinymist.git synced 2025-08-04 10:18:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

feat: track fine-grained revisions of font, registry, entry, and vfs (#1192)

Browse source 
* feat: set flag to indicate whether we are compiling files

g1

dev: stateless compile

dev: vfs revise apis

g1

feat: bump revision on state changes

feat: track font and package changes

dev: some cases that can change state of cache

changes

* feat: implement shared source cache

* fix: take db state

* dev: update take state location

* fix: example
This commit is contained in:
Myriad-Dreamin 2025-01-19 18:23:41 +08:00 • committed by GitHub
parent 8481b77e3c
commit 1f01ec1f6c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
24 changed files with 841 additions and 484 deletions
Download patch file Download diff file Expand all files Collapse all files

4
crates/tinymist-project/src/watch.rs
View file

@ -17,9 +17,9 @@ use tokio::sync::mpsc;
use typst::diag::FileError;
use crate::vfs::{
notify::{FileChangeSet, FileSnapshot, FilesystemEvent, NotifyMessage, UpstreamUpdateEvent},
notify::{FilesystemEvent, NotifyMessage, UpstreamUpdateEvent},
system::SystemAccessModel,
PathAccessModel,
FileChangeSet, FileSnapshot, PathAccessModel,
};
type WatcherPair = (RecommendedWatcher, mpsc::UnboundedReceiver<NotifyEvent>);

4
crates/tinymist-project/src/world.rs
View file

@ -7,7 +7,9 @@ pub use tinymist_world::config::CompileFontOpts;
use tinymist_world::package::RegistryPathMapper;
pub use tinymist_world::vfs;
pub use tinymist_world::{entry::*, EntryOpts, EntryState};
pub use tinymist_world::{font, package, CompilerUniverse, CompilerWorld, Revising, TaskInputs};
pub use tinymist_world::{
font, package, CompilerUniverse, CompilerWorld, RevisingUniverse, TaskInputs,
};
use std::path::Path;
use std::{borrow::Cow, sync::Arc};

2
crates/tinymist-query/src/code_context.rs
View file

@ -106,7 +106,7 @@ impl SemanticRequest for InteractCodeContextRequest {
mapped_source.id(),
Bytes::from(mapped_source.text().as_bytes()),
);
world.source_db.take_state();
world.take_db();
let root = LinkedNode::new(mapped_source.root());
let leaf = root.leaf_at_compat(start + offset)?;

2
crates/tinymist-query/src/docs/convert.rs
View file

@ -25,7 +25,7 @@ pub(crate) fn convert_docs(ctx: &SharedContext, content: &str) -> StrResult<EcoS
});
w.map_shadow_by_id(w.main(), Bytes::from(content.as_bytes().to_owned()))?;
// todo: bad performance
w.source_db.take_state();
w.take_db();
let conv = typlite::Typlite::new(Arc::new(w))
.with_library(DOCS_LIB.clone())

26
crates/tinymist-query/src/tests.rs
View file

@ -45,8 +45,8 @@ pub fn snapshot_testing(name: &str, f: &impl Fn(&mut LocalContext, PathBuf)) {
#[cfg(windows)]
let contents = contents.replace("\r\n", "\n");
run_with_sources(&contents, |world, path| {
run_with_ctx(world, path, f);
run_with_sources(&contents, |verse, path| {
run_with_ctx(verse, path, f);
});
});
});
@ -69,7 +69,8 @@ pub fn run_with_ctx<T>(
})
.collect::<Vec<_>>();
let world = verse.snapshot();
let mut world = verse.snapshot();
world.set_is_compiling(false);
let source = world.source_by_path(&path).ok().unwrap();
let docs = find_module_level_docs(&source).unwrap_or_default();
@ -120,24 +121,25 @@ pub fn compile_doc_for_test(
ctx: &mut LocalContext,
properties: &HashMap<&str, &str>,
) -> Option<VersionedDocument> {
let entry = match properties.get("compile")?.trim() {
"true" => ctx.world.entry_state(),
let prev = ctx.world.entry_state();
let next = match properties.get("compile")?.trim() {
"true" => prev.clone(),
"false" => return None,
path if path.ends_with(".typ") => {
ctx.world.entry_state().select_in_workspace(Path::new(path))
}
path if path.ends_with(".typ") => prev.select_in_workspace(Path::new(path)),
v => panic!("invalid value for 'compile' property: {v}"),
};
let mut world = Cow::Borrowed(&ctx.world);
if entry != ctx.world.entry_state() {
if next != prev {
world = Cow::Owned(world.task(TaskInputs {
entry: Some(entry),
entry: Some(next),
..Default::default()
}));
}
let mut world = world.into_owned();
world.set_is_compiling(true);
let doc = typst::compile(world.as_ref()).output.unwrap();
let doc = typst::compile(&world).output.unwrap();
Some(VersionedDocument {
version: 0,
document: Arc::new(doc),
@ -190,8 +192,6 @@ pub fn run_with_sources<T>(source: &str, f: impl FnOnce(&mut LspUniverse, PathBu
last_pw = Some(pw);
}
verse.mutate_entry(EntryState::new_detached()).unwrap();
let pw = last_pw.unwrap();
verse
.mutate_entry(EntryState::new_rooted(

5
crates/tinymist-vfs/src/dummy.rs
View file

@ -1,5 +1,6 @@
use std::path::Path;
use tinymist_std::ImmutPath;
use typst::diag::{FileError, FileResult};
use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
@ -13,8 +14,8 @@ use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
pub struct DummyAccessModel;
impl AccessModel for DummyAccessModel {
fn content(&self, _src: TypstFileId) -> FileResult<Bytes> {
Err(FileError::AccessDenied)
fn content(&self, _src: TypstFileId) -> (Option<ImmutPath>, FileResult<Bytes>) {
(None, Err(FileError::AccessDenied))
}
}

445
crates/tinymist-vfs/src/lib.rs
View file

@ -19,9 +19,16 @@ pub mod system;
/// [`Vfs`] will make a overlay access model over the provided dummy access
/// model.
pub mod dummy;
/// Provides snapshot models
pub mod snapshot;
pub use snapshot::*;
use tinymist_std::hash::{FxDashMap, FxHashMap};
/// Provides notify access model which retrieves file system events and changes
/// from some notify backend.
pub mod notify;
pub use notify::{FilesystemEvent, MemoryEvent};
/// Provides overlay access model which allows to shadow the underlying access
/// model with memory contents.
pub mod overlay;
@ -32,23 +39,27 @@ pub mod trace;
mod utils;
mod path_mapper;
use notify::{FilesystemEvent, NotifyAccessModel};
pub use path_mapper::{PathResolution, RootResolver, WorkspaceResolution, WorkspaceResolver};
use resolve::ResolveAccessModel;
use rpds::RedBlackTreeMapSync;
pub use typst::foundations::Bytes;
pub use typst::syntax::FileId as TypstFileId;
pub use tinymist_std::time::Time;
pub use tinymist_std::ImmutPath;
use typst::syntax::Source;
use core::fmt;
use std::num::NonZeroUsize;
use std::sync::OnceLock;
use std::{path::Path, sync::Arc};
use parking_lot::RwLock;
use parking_lot::{Mutex, RwLock};
use typst::diag::{FileError, FileResult};
use self::overlay::OverlayAccessModel;
use crate::notify::NotifyAccessModel;
use crate::overlay::OverlayAccessModel;
use crate::resolve::ResolveAccessModel;
/// Handle to a file in [`Vfs`]
pub type FileId = TypstFileId;
@ -62,7 +73,7 @@ pub trait PathAccessModel {
///
/// This is called when the vfs is reset. See [`Vfs`]'s reset method for
/// more information.
fn clear(&mut self) {}
fn reset(&mut self) {}
/// Return the content of a file entry.
fn content(&self, src: &Path) -> FileResult<Bytes>;
@ -77,10 +88,10 @@ pub trait AccessModel {
///
/// This is called when the vfs is reset. See [`Vfs`]'s reset method for
/// more information.
fn clear(&mut self) {}
fn reset(&mut self) {}
/// Return the content of a file entry.
fn content(&self, src: TypstFileId) -> FileResult<Bytes>;
fn content(&self, src: TypstFileId) -> (Option<ImmutPath>, FileResult<Bytes>);
}
#[derive(Clone)]
@ -100,8 +111,9 @@ impl<M> PathAccessModel for SharedAccessModel<M>
where
M: PathAccessModel,
{
fn clear(&mut self) {
self.inner.write().clear();
#[inline]
fn reset(&mut self) {
self.inner.write().reset();
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
@ -119,11 +131,55 @@ pub trait FsProvider {
fn file_path(&self, id: TypstFileId) -> FileResult<PathResolution>;
fn read(&self, id: TypstFileId) -> FileResult<Bytes>;
fn read_source(&self, id: TypstFileId) -> FileResult<Source>;
}
struct SourceEntry {
last_accessed_rev: NonZeroUsize,
source: FileResult<Source>,
}
#[derive(Default)]
struct SourceIdShard {
last_accessed_rev: usize,
recent_source: Option<Source>,
sources: FxHashMap<Bytes, SourceEntry>,
}
#[derive(Default, Clone)]
pub struct SourceCache {
/// The cache entries for each paths
cache_entries: Arc<FxDashMap<TypstFileId, SourceIdShard>>,
}
impl SourceCache {
pub fn evict(&self, curr: NonZeroUsize, threshold: usize) {
self.cache_entries.retain(|_, shard| {
let diff = curr.get().saturating_sub(shard.last_accessed_rev);
if diff > threshold {
return false;
}
shard.sources.retain(|_, entry| {
let diff = curr.get().saturating_sub(entry.last_accessed_rev.get());
diff <= threshold
});
true
});
}
}
/// Create a new `Vfs` harnessing over the given `access_model` specific for
/// `reflexo_world::CompilerWorld`. With vfs, we can minimize the
/// implementation overhead for [`AccessModel`] trait.
pub struct Vfs<M: PathAccessModel + Sized> {
source_cache: SourceCache,
// The slots for all the files during a single lifecycle.
// pub slots: Arc<Mutex<FxHashMap<TypstFileId, SourceCache>>>,
managed: Arc<Mutex<EntryMap>>,
paths: Arc<Mutex<PathMap>>,
revision: NonZeroUsize,
// access_model: TraceAccessModel<VfsAccessModel<M>>,
/// The wrapped access model.
access_model: VfsAccessModel<M>,
@ -136,8 +192,16 @@ impl<M: PathAccessModel + Sized> fmt::Debug for Vfs<M> {
}
impl<M: PathAccessModel + Clone + Sized> Vfs<M> {
pub fn revision(&self) -> NonZeroUsize {
self.revision
}
pub fn snapshot(&self) -> Self {
Self {
source_cache: self.source_cache.clone(),
managed: self.managed.clone(),
paths: self.paths.clone(),
revision: self.revision,
access_model: self.access_model.clone(),
}
}
@ -169,18 +233,46 @@ impl<M: PathAccessModel + Sized> Vfs<M> {
// If you want to trace the access model, uncomment the following line
// let access_model = TraceAccessModel::new(access_model);
Self { access_model }
Self {
source_cache: SourceCache::default(),
managed: Arc::default(),
paths: Arc::default(),
revision: NonZeroUsize::new(1).expect("initial revision is 1"),
access_model,
}
}
/// Reset the source file and path references.
///
/// It performs a rolling reset, with discard some cache file entry when it
/// is unused in recent 30 lifecycles.
///
/// Note: The lifetime counter is incremented every time this function is
/// called.
pub fn reset(&mut self) {
self.access_model.clear();
/// Reset all state.
pub fn reset_all(&mut self) {
self.reset_access_model();
self.reset_cache();
self.take_state();
}
/// Reset access model.
pub fn reset_access_model(&mut self) {
self.access_model.reset();
}
/// Reset all possible caches.
pub fn reset_cache(&mut self) {
self.revise().reset_cache();
}
/// Clear the cache that is not touched for a long time.
pub fn evict(&mut self, threshold: usize) {
let mut m = self.managed.lock();
let rev = self.revision.get();
for (id, entry) in m.entries.clone().iter() {
let entry_rev = entry.bytes.get().map(|e| e.1).unwrap_or_default();
if entry_rev + threshold < rev {
m.entries.remove_mut(id);
}
}
}
pub fn take_state(&mut self) -> SourceCache {
std::mem::take(&mut self.source_cache)
}
/// Resolve the real path for a file id.
@ -188,67 +280,288 @@ impl<M: PathAccessModel + Sized> Vfs<M> {
self.access_model.inner.resolver.path_for_id(id)
}
/// Reset the shadowing files in [`OverlayAccessModel`].
///
/// Note: This function is independent from [`Vfs::reset`].
pub fn reset_shadow(&mut self) {
self.access_model.clear_shadow();
self.access_model.inner.inner.clear_shadow();
}
/// Get paths to all the shadowing files in [`OverlayAccessModel`].
/// Get paths to all the shadowing paths in [`OverlayAccessModel`].
pub fn shadow_paths(&self) -> Vec<ImmutPath> {
self.access_model.inner.inner.file_paths()
}
/// Get paths to all the shadowing files in [`OverlayAccessModel`].
/// Get paths to all the shadowing file ids in [`OverlayAccessModel`].
///
/// The in memory untitled files can have no path so
/// they only have file ids.
pub fn shadow_ids(&self) -> Vec<TypstFileId> {
self.access_model.file_paths()
}
/// Add a shadowing file to the [`OverlayAccessModel`].
pub fn map_shadow(&mut self, path: &Path, content: Bytes) -> FileResult<()> {
self.access_model
.inner
.inner
.add_file(path, content, |c| c.into());
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`].
pub fn remove_shadow(&mut self, path: &Path) {
self.access_model.inner.inner.remove_file(path);
}
/// Add a shadowing file to the [`OverlayAccessModel`] by file id.
pub fn map_shadow_by_id(&mut self, file_id: TypstFileId, content: Bytes) -> FileResult<()> {
self.access_model.add_file(&file_id, content, |c| *c);
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`] by file id.
pub fn remove_shadow_by_id(&mut self, file_id: TypstFileId) {
self.access_model.remove_file(&file_id);
}
/// Let the vfs notify the access model with a filesystem event.
///
/// See [`NotifyAccessModel`] for more information.
pub fn notify_fs_event(&mut self, event: FilesystemEvent) {
self.access_model.inner.inner.inner.notify(event);
}
/// Returns the overall memory usage for the stored files.
pub fn memory_usage(&self) -> usize {
0
}
/// Read a file.
pub fn read(&self, path: TypstFileId) -> FileResult<Bytes> {
self.access_model.content(path)
pub fn revise(&mut self) -> RevisingVfs<M> {
let managed = self.managed.lock().clone();
let paths = self.paths.lock().clone();
RevisingVfs {
managed,
paths,
inner: self,
view_changed: false,
}
}
/// Reads a file.
pub fn read(&self, fid: TypstFileId) -> FileResult<Bytes> {
let bytes = self.managed.lock().slot(fid, |entry| entry.bytes.clone());
self.read_content(&bytes, fid).clone()
}
/// Reads a source.
pub fn source(&self, file_id: TypstFileId) -> FileResult<Source> {
let (bytes, source) = self
.managed
.lock()
.slot(file_id, |entry| (entry.bytes.clone(), entry.source.clone()));
let source = source.get_or_init(|| {
let content = self
.read_content(&bytes, file_id)
.as_ref()
.map_err(Clone::clone)?;
let mut cache_entry = self.source_cache.cache_entries.entry(file_id).or_default();
if let Some(source) = cache_entry.sources.get(content) {
return source.source.clone();
}
let source = (|| {
let prev = cache_entry.recent_source.clone();
let content = from_utf8_or_bom(content).map_err(|_| FileError::InvalidUtf8)?;
let next = match prev {
Some(mut prev) => {
prev.replace(content);
prev
}
None => Source::new(file_id, content.to_owned()),
};
let should_update = cache_entry.recent_source.is_none()
|| cache_entry.last_accessed_rev < self.revision.get();
if should_update {
cache_entry.recent_source = Some(next.clone());
}
Ok(next)
})();
let entry = cache_entry
.sources
.entry(content.clone())
.or_insert_with(|| SourceEntry {
last_accessed_rev: self.revision,
source: source.clone(),
});
if entry.last_accessed_rev < self.revision {
entry.last_accessed_rev = self.revision;
}
source
});
source.clone()
}
/// Reads and caches content of a file.
fn read_content<'a>(&self, bytes: &'a BytesQuery, fid: TypstFileId) -> &'a FileResult<Bytes> {
&bytes
.get_or_init(|| {
let (path, content) = self.access_model.content(fid);
if let Some(path) = path.as_ref() {
self.paths.lock().insert(path, fid);
}
(path, self.revision.get(), content)
})
.2
}
}
pub struct RevisingVfs<'a, M: PathAccessModel + Sized> {
inner: &'a mut Vfs<M>,
managed: EntryMap,
paths: PathMap,
view_changed: bool,
}
impl<M: PathAccessModel + Sized> Drop for RevisingVfs<'_, M> {
fn drop(&mut self) {
if self.view_changed {
self.inner.managed = Arc::new(Mutex::new(std::mem::take(&mut self.managed)));
self.inner.paths = Arc::new(Mutex::new(std::mem::take(&mut self.paths)));
let revision = &mut self.inner.revision;
*revision = revision.checked_add(1).expect("revision overflowed");
}
}
}
impl<M: PathAccessModel + Sized> RevisingVfs<'_, M> {
pub fn vfs(&mut self) -> &mut Vfs<M> {
self.inner
}
fn am(&mut self) -> &mut VfsAccessModel<M> {
&mut self.inner.access_model
}
fn invalidate_path(&mut self, path: &Path) {
if let Some(fids) = self.paths.remove(path) {
self.view_changed = true;
for fid in fids {
self.invalidate_file_id(fid);
}
}
}
fn invalidate_file_id(&mut self, file_id: TypstFileId) {
self.view_changed = true;
self.managed.slot(file_id, |e| {
e.bytes = Arc::default();
e.source = Arc::default();
});
}
/// Reset the shadowing files in [`OverlayAccessModel`].
///
/// Note: This function is independent from [`Vfs::reset`].
pub fn reset_shadow(&mut self) {
for path in self.am().inner.inner.file_paths() {
self.invalidate_path(&path);
}
for fid in self.am().file_paths() {
self.invalidate_file_id(fid);
}
self.am().clear_shadow();
self.am().inner.inner.clear_shadow();
}
/// Reset all caches. This can happen when:
/// - package paths are reconfigured.
/// - The root of the workspace is switched.
pub fn reset_cache(&mut self) {
self.view_changed = true;
self.managed = EntryMap::default();
self.paths = PathMap::default();
}
/// Add a shadowing file to the [`OverlayAccessModel`].
pub fn map_shadow(&mut self, path: &Path, snap: FileSnapshot) -> FileResult<()> {
self.view_changed = true;
self.invalidate_path(path);
self.am().inner.inner.add_file(path, snap, |c| c.into());
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`].
pub fn unmap_shadow(&mut self, path: &Path) -> FileResult<()> {
self.view_changed = true;
self.invalidate_path(path);
self.am().inner.inner.remove_file(path);
Ok(())
}
/// Add a shadowing file to the [`OverlayAccessModel`] by file id.
pub fn map_shadow_by_id(&mut self, file_id: TypstFileId, snap: FileSnapshot) -> FileResult<()> {
self.view_changed = true;
self.invalidate_file_id(file_id);
self.am().add_file(&file_id, snap, |c| *c);
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`] by file id.
pub fn remove_shadow_by_id(&mut self, file_id: TypstFileId) {
self.view_changed = true;
self.invalidate_file_id(file_id);
self.am().remove_file(&file_id);
}
/// Let the vfs notify the access model with a filesystem event.
///
/// See [`NotifyAccessModel`] for more information.
pub fn notify_fs_event(&mut self, event: FilesystemEvent) {
self.notify_fs_changes(event.split().0);
}
/// Let the vfs notify the access model with a filesystem changes.
///
/// See [`NotifyAccessModel`] for more information.
pub fn notify_fs_changes(&mut self, event: FileChangeSet) {
self.view_changed = true;
self.am().inner.inner.inner.notify(event);
}
}
type BytesQuery = Arc<OnceLock<(Option<ImmutPath>, usize, FileResult<Bytes>)>>;
#[derive(Debug, Clone, Default)]
struct VfsEntry {
bytes: BytesQuery,
source: Arc<OnceLock<FileResult<Source>>>,
}
#[derive(Clone, Default)]
struct EntryMap {
entries: RedBlackTreeMapSync<TypstFileId, VfsEntry>,
}
impl EntryMap {
/// Read a slot.
#[inline(always)]
fn slot<T>(&mut self, path: TypstFileId, f: impl FnOnce(&mut VfsEntry) -> T) -> T {
if let Some(entry) = self.entries.get_mut(&path) {
f(entry)
} else {
let mut entry = VfsEntry::default();
let res = f(&mut entry);
self.entries.insert_mut(path, entry);
res
}
}
}
#[derive(Clone, Default)]
struct PathMap {
paths: FxHashMap<ImmutPath, Vec<TypstFileId>>,
}
impl PathMap {
fn insert(&mut self, path: &ImmutPath, fid: TypstFileId) {
if let Some(fids) = self.paths.get_mut(path) {
fids.push(fid);
} else {
self.paths.insert(path.clone(), vec![fid]);
}
}
fn remove(&mut self, path: &Path) -> Option<Vec<TypstFileId>> {
self.paths.remove(path)
}
}
/// Convert a byte slice to a string, removing UTF-8 BOM if present.
fn from_utf8_or_bom(buf: &[u8]) -> FileResult<&str> {
Ok(std::str::from_utf8(if buf.starts_with(b"\xef\xbb\xbf") {
// remove UTF-8 BOM
&buf[3..]
} else {
// Assume UTF-8
buf
})?)
}
#[cfg(test)]

154
crates/tinymist-vfs/src/notify.rs
View file

@ -1,118 +1,12 @@
use core::fmt;
use std::path::Path;
use rpds::RedBlackTreeMapSync;
use typst::diag::{FileError, FileResult};
use typst::diag::FileResult;
use crate::{Bytes, ImmutPath, PathAccessModel};
/// A file snapshot that is notified by some external source
///
/// Note: The error is boxed to avoid large stack size
#[derive(Clone, PartialEq, Eq)]
pub struct FileSnapshot(Result<Bytes, Box<FileError>>);
impl fmt::Debug for FileSnapshot {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.0.as_ref() {
Ok(v) => f
.debug_struct("FileSnapshot")
.field("content", &FileContent { len: v.len() })
.finish(),
Err(e) => f.debug_struct("FileSnapshot").field("error", &e).finish(),
}
}
}
impl FileSnapshot {
/// content of the file
#[inline]
#[track_caller]
pub fn content(&self) -> FileResult<&Bytes> {
self.0.as_ref().map_err(|e| *e.clone())
}
/// Whether the related file is a file
#[inline]
#[track_caller]
pub fn is_file(&self) -> FileResult<bool> {
self.content().map(|_| true)
}
}
impl std::ops::Deref for FileSnapshot {
type Target = Result<Bytes, Box<FileError>>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for FileSnapshot {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
/// Convenient function to create a [`FileSnapshot`] from tuple
impl From<FileResult<Bytes>> for FileSnapshot {
fn from(result: FileResult<Bytes>) -> Self {
Self(result.map_err(Box::new))
}
}
/// A set of changes to the filesystem.
///
/// The correct order of applying changes is:
/// 1. Remove files
/// 2. Upsert (Insert or Update) files
#[derive(Debug, Clone, Default)]
pub struct FileChangeSet {
/// Files to remove
pub removes: Vec<ImmutPath>,
/// Files to insert or update
pub inserts: Vec<(ImmutPath, FileSnapshot)>,
}
impl FileChangeSet {
/// Create a new empty changeset
pub fn is_empty(&self) -> bool {
self.inserts.is_empty() && self.removes.is_empty()
}
/// Create a new changeset with removing files
pub fn new_removes(removes: Vec<ImmutPath>) -> Self {
Self {
removes,
inserts: vec![],
}
}
/// Create a new changeset with inserting files
pub fn new_inserts(inserts: Vec<(ImmutPath, FileSnapshot)>) -> Self {
Self {
removes: vec![],
inserts,
}
}
/// Utility function to insert a possible file to insert or update
pub fn may_insert(&mut self, v: Option<(ImmutPath, FileSnapshot)>) {
if let Some(v) = v {
self.inserts.push(v);
}
}
/// Utility function to insert multiple possible files to insert or update
pub fn may_extend(&mut self, v: Option<impl Iterator<Item = (ImmutPath, FileSnapshot)>>) {
if let Some(v) = v {
self.inserts.extend(v);
}
}
}
use crate::{Bytes, FileChangeSet, FileSnapshot, ImmutPath, PathAccessModel};
/// A memory event that is notified by some external source
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum MemoryEvent {
/// Reset all dependencies and update according to the given changeset
///
@ -121,7 +15,7 @@ pub enum MemoryEvent {
/// this:
///
/// ```
/// use tinymist_vfs::notify::{MemoryEvent, FileChangeSet};
/// use tinymist_vfs::{FileChangeSet, notify::MemoryEvent};
/// let event = MemoryEvent::Sync(FileChangeSet::default());
/// ```
Sync(FileChangeSet),
@ -156,6 +50,18 @@ pub enum FilesystemEvent {
},
}
impl FilesystemEvent {
pub fn split(self) -> (FileChangeSet, Option<UpstreamUpdateEvent>) {
match self {
FilesystemEvent::UpstreamUpdate {
changeset,
upstream_event,
} => (changeset, upstream_event),
FilesystemEvent::Update(changeset) => (changeset, None),
}
}
}
/// A message that is sent to some file watcher
#[derive(Debug)]
pub enum NotifyMessage {
@ -211,23 +117,23 @@ impl<M: PathAccessModel> NotifyAccessModel<M> {
}
/// Notify the access model with a filesystem event
pub fn notify(&mut self, event: FilesystemEvent) {
match event {
FilesystemEvent::UpstreamUpdate { changeset, .. }
| FilesystemEvent::Update(changeset) => {
for path in changeset.removes {
self.files.remove_mut(&path);
}
pub fn notify(&mut self, changeset: FileChangeSet) {
for path in changeset.removes {
self.files.remove_mut(&path);
}
for (path, contents) in changeset.inserts {
self.files.insert_mut(path, contents);
}
}
for (path, contents) in changeset.inserts {
self.files.insert_mut(path, contents);
}
}
}
impl<M: PathAccessModel> PathAccessModel for NotifyAccessModel<M> {
#[inline]
fn reset(&mut self) {
self.inner.reset();
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
if let Some(entry) = self.files.get(src) {
return entry.content().cloned();
@ -236,9 +142,3 @@ impl<M: PathAccessModel> PathAccessModel for NotifyAccessModel<M> {
self.inner.content(src)
}
}
#[derive(Debug)]
#[allow(dead_code)]
struct FileContent {
len: usize,
}

26
crates/tinymist-vfs/src/overlay.rs
View file

@ -4,13 +4,13 @@ use rpds::RedBlackTreeMapSync;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
use crate::{AccessModel, Bytes, FileSnapshot, PathAccessModel, TypstFileId};
/// Provides overlay access model which allows to shadow the underlying access
/// model with memory contents.
#[derive(Default, Debug, Clone)]
pub struct OverlayAccessModel<K: Ord, M> {
files: RedBlackTreeMapSync<K, Bytes>,
files: RedBlackTreeMapSync<K, FileSnapshot>,
/// The underlying access model
pub inner: M,
}
@ -45,16 +45,20 @@ impl<K: Ord + Clone, M> OverlayAccessModel<K, M> {
}
/// Add a shadow file to the [`OverlayAccessModel`]
pub fn add_file<Q: Ord + ?Sized>(&mut self, path: &Q, content: Bytes, cast: impl Fn(&Q) -> K)
where
pub fn add_file<Q: Ord + ?Sized>(
&mut self,
path: &Q,
snap: FileSnapshot,
cast: impl Fn(&Q) -> K,
) where
K: Borrow<Q>,
{
match self.files.get_mut(path) {
Some(e) => {
*e = content;
*e = snap;
}
None => {
self.files.insert_mut(cast(path), content);
self.files.insert_mut(cast(path), snap);
}
}
}
@ -71,7 +75,7 @@ impl<K: Ord + Clone, M> OverlayAccessModel<K, M> {
impl<M: PathAccessModel> PathAccessModel for OverlayAccessModel<ImmutPath, M> {
fn content(&self, src: &Path) -> FileResult<Bytes> {
if let Some(content) = self.files.get(src) {
return Ok(content.clone());
return content.content().cloned();
}
self.inner.content(src)
@ -79,9 +83,13 @@ impl<M: PathAccessModel> PathAccessModel for OverlayAccessModel<ImmutPath, M> {
}
impl<M: AccessModel> AccessModel for OverlayAccessModel<TypstFileId, M> {
fn content(&self, src: TypstFileId) -> FileResult<Bytes> {
fn reset(&mut self) {
self.inner.reset();
}
fn content(&self, src: TypstFileId) -> (Option<ImmutPath>, FileResult<Bytes>) {
if let Some(content) = self.files.get(&src) {
return Ok(content.clone());
return (None, content.content().cloned());
}
self.inner.content(src)

16
crates/tinymist-vfs/src/resolve.rs
View file

@ -1,5 +1,6 @@
use std::{fmt::Debug, sync::Arc};
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{path_mapper::RootResolver, AccessModel, Bytes, PathAccessModel, TypstFileId};
@ -18,8 +19,17 @@ impl<M> Debug for ResolveAccessModel<M> {
}
impl<M: PathAccessModel> AccessModel for ResolveAccessModel<M> {
fn content(&self, fid: TypstFileId) -> FileResult<Bytes> {
self.inner
.content(&self.resolver.path_for_id(fid)?.to_err()?)
#[inline]
fn reset(&mut self) {
self.inner.reset();
}
fn content(&self, fid: TypstFileId) -> (Option<ImmutPath>, FileResult<Bytes>) {
let resolved = Ok(()).and_then(|_| self.resolver.path_for_id(fid)?.to_err());
match resolved {
Ok(path) => (Some(path.as_path().into()), self.inner.content(&path)),
Err(e) => (None, Err(e)),
}
}
}

116
crates/tinymist-vfs/src/snapshot.rs Normal file
View file

@ -0,0 +1,116 @@
use core::fmt;
use typst::diag::{FileError, FileResult};
use crate::{Bytes, ImmutPath};
/// A file snapshot that is notified by some external source
///
/// Note: The error is boxed to avoid large stack size
#[derive(Clone, PartialEq, Eq)]
pub struct FileSnapshot(Result<Bytes, Box<FileError>>);
#[derive(Debug)]
#[allow(dead_code)]
struct FileContent {
len: usize,
}
impl fmt::Debug for FileSnapshot {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.0.as_ref() {
Ok(v) => f
.debug_struct("FileSnapshot")
.field("content", &FileContent { len: v.len() })
.finish(),
Err(e) => f.debug_struct("FileSnapshot").field("error", &e).finish(),
}
}
}
impl FileSnapshot {
/// content of the file
#[inline]
#[track_caller]
pub fn content(&self) -> FileResult<&Bytes> {
self.0.as_ref().map_err(|e| *e.clone())
}
/// Whether the related file is a file
#[inline]
#[track_caller]
pub fn is_file(&self) -> FileResult<bool> {
self.content().map(|_| true)
}
}
impl std::ops::Deref for FileSnapshot {
type Target = Result<Bytes, Box<FileError>>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl std::ops::DerefMut for FileSnapshot {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
/// Convenient function to create a [`FileSnapshot`] from tuple
impl From<FileResult<Bytes>> for FileSnapshot {
fn from(result: FileResult<Bytes>) -> Self {
Self(result.map_err(Box::new))
}
}
/// A set of changes to the filesystem.
///
/// The correct order of applying changes is:
/// 1. Remove files
/// 2. Upsert (Insert or Update) files
#[derive(Debug, Clone, Default)]
pub struct FileChangeSet {
/// Files to remove
pub removes: Vec<ImmutPath>,
/// Files to insert or update
pub inserts: Vec<(ImmutPath, FileSnapshot)>,
}
impl FileChangeSet {
/// Create a new empty changeset
pub fn is_empty(&self) -> bool {
self.inserts.is_empty() && self.removes.is_empty()
}
/// Create a new changeset with removing files
pub fn new_removes(removes: Vec<ImmutPath>) -> Self {
Self {
removes,
inserts: vec![],
}
}
/// Create a new changeset with inserting files
pub fn new_inserts(inserts: Vec<(ImmutPath, FileSnapshot)>) -> Self {
Self {
removes: vec![],
inserts,
}
}
/// Utility function to insert a possible file to insert or update
pub fn may_insert(&mut self, v: Option<(ImmutPath, FileSnapshot)>) {
if let Some(v) = v {
self.inserts.push(v);
}
}
/// Utility function to insert multiple possible files to insert or update
pub fn may_extend(&mut self, v: Option<impl Iterator<Item = (ImmutPath, FileSnapshot)>>) {
if let Some(v) = v {
self.inserts.extend(v);
}
}
}

8
crates/tinymist-vfs/src/trace.rs
View file

@ -1,5 +1,6 @@
use std::sync::atomic::AtomicU64;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{AccessModel, Bytes, TypstFileId};
@ -25,11 +26,12 @@ impl<M: AccessModel + Sized> TraceAccessModel<M> {
}
impl<M: AccessModel + Sized> AccessModel for TraceAccessModel<M> {
fn clear(&mut self) {
self.inner.clear();
#[inline]
fn reset(&mut self) {
self.inner.reset();
}
fn content(&self, src: TypstFileId) -> FileResult<Bytes> {
fn content(&self, src: TypstFileId) -> (Option<ImmutPath>, FileResult<Bytes>) {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.content(src);
let elapsed = instant.elapsed();

4
crates/tinymist-world/src/entry.rs
View file

@ -16,10 +16,6 @@ pub trait EntryReader {
}
pub trait EntryManager: EntryReader {
fn reset(&mut self) -> SourceResult<()> {
Ok(())
}
fn mutate_entry(&mut self, state: EntryState) -> SourceResult<EntryState>;
}

5
crates/tinymist-world/src/font/resolver.rs
View file

@ -1,6 +1,7 @@
use core::fmt;
use std::{
collections::HashMap,
num::NonZeroUsize,
path::PathBuf,
sync::{Arc, Mutex},
};
@ -16,6 +17,10 @@ use crate::Bytes;
/// It also reuse FontBook for font-related query.
/// The index is the index of the font in the `FontBook.infos`.
pub trait FontResolver {
fn revision(&self) -> Option<NonZeroUsize> {
None
}
fn font_book(&self) -> &LazyHash<FontBook>;
fn font(&self, idx: usize) -> Option<Font>;

2
crates/tinymist-world/src/lib.rs
View file

@ -133,7 +133,7 @@ pub trait WorldDeps {
}
/// type trait interface of [`CompilerWorld`].
pub trait CompilerFeat {
pub trait CompilerFeat: Send + Sync + 'static {
/// Specify the font resolver for typst compiler.
type FontResolver: FontResolver + Send + Sync + Sized;
/// Specify the access model for VFS.

5
crates/tinymist-world/src/package/mod.rs
View file

@ -1,4 +1,5 @@
impl Notifier for DummyNotifier {}
use std::num::NonZeroUsize;
use std::{path::Path, sync::Arc};
use ecow::EcoString;
@ -18,6 +19,10 @@ pub mod http;
pub trait PackageRegistry {
fn reset(&mut self) {}
fn revision(&self) -> Option<NonZeroUsize> {
None
}
fn resolve(&self, spec: &PackageSpec) -> Result<Arc<Path>, PackageError>;
/// A list of all available packages and optionally descriptions for them.

131
crates/tinymist-world/src/source.rs
View file

@ -3,7 +3,7 @@
use core::fmt;
use std::{num::NonZeroUsize, sync::Arc};
use parking_lot::{Mutex, RwLock};
use parking_lot::Mutex;
use tinymist_std::hash::FxHashMap;
use tinymist_std::QueryRef;
use tinymist_vfs::{Bytes, FsProvider, TypstFileId};
@ -22,40 +22,8 @@ pub trait Revised {
fn last_accessed_rev(&self) -> NonZeroUsize;
}
pub struct SharedState<T> {
pub committed_revision: Option<usize>,
// todo: fine-grained lock
/// The cache entries for each paths
cache_entries: FxHashMap<TypstFileId, T>,
}
impl<T> fmt::Debug for SharedState<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SharedState")
.field("committed_revision", &self.committed_revision)
.finish()
}
}
impl<T> Default for SharedState<T> {
fn default() -> Self {
SharedState {
committed_revision: None,
cache_entries: FxHashMap::default(),
}
}
}
impl<T: Revised> SharedState<T> {
fn gc(&mut self) {
let committed = self.committed_revision.unwrap_or(0);
self.cache_entries
.retain(|_, v| committed.saturating_sub(v.last_accessed_rev().get()) <= 30);
}
}
pub struct SourceCache {
last_accessed_rev: NonZeroUsize,
touched_by_compile: bool,
fid: TypstFileId,
source: IncrFileQuery<Source>,
buffer: FileQuery<Bytes>,
@ -67,42 +35,9 @@ impl fmt::Debug for SourceCache {
}
}
impl Revised for SourceCache {
fn last_accessed_rev(&self) -> NonZeroUsize {
self.last_accessed_rev
}
}
pub struct SourceState {
pub revision: NonZeroUsize,
pub slots: Arc<Mutex<FxHashMap<TypstFileId, SourceCache>>>,
}
impl SourceState {
pub fn commit_impl(self, state: &mut SharedState<SourceCache>) {
log::debug!("drop source db revision {}", self.revision);
if let Ok(slots) = Arc::try_unwrap(self.slots) {
// todo: utilize the committed revision is not zero
if state
.committed_revision
.is_some_and(|committed| committed >= self.revision.get())
{
return;
}
log::debug!("committing source db revision {}", self.revision);
state.committed_revision = Some(self.revision.get());
state.cache_entries = slots.into_inner();
state.gc();
}
}
}
#[derive(Clone)]
pub struct SourceDb {
pub revision: NonZeroUsize,
pub shared: Arc<RwLock<SharedState<SourceCache>>>,
pub is_compiling: bool,
/// The slots for all the files during a single lifecycle.
pub slots: Arc<Mutex<FxHashMap<TypstFileId, SourceCache>>>,
}
@ -114,11 +49,8 @@ impl fmt::Debug for SourceDb {
}
impl SourceDb {
pub fn take_state(&mut self) -> SourceState {
SourceState {
revision: self.revision,
slots: std::mem::take(&mut self.slots),
}
pub fn set_is_compiling(&mut self, is_compiling: bool) {
self.is_compiling = is_compiling;
}
/// Returns the overall memory usage for the stored files.
@ -152,8 +84,11 @@ impl SourceDb {
/// When you don't reset the vfs for each compilation, this function will
/// still return remaining files from the previous compilation.
pub fn iter_dependencies_dyn(&self, f: &mut dyn FnMut(TypstFileId)) {
for slot in self.slots.lock().iter() {
f(slot.1.fid);
for slot in self.slots.lock().values() {
if !slot.touched_by_compile {
continue;
}
f(slot.fid);
}
}
@ -190,31 +125,29 @@ impl SourceDb {
/// Insert a new slot into the vfs.
fn slot<T>(&self, fid: TypstFileId, f: impl FnOnce(&SourceCache) -> T) -> T {
let mut slots = self.slots.lock();
f(slots.entry(fid).or_insert_with(|| {
let state = self.shared.read();
let cache_entry = state.cache_entries.get(&fid);
f({
let entry = slots.entry(fid).or_insert_with(|| SourceCache {
touched_by_compile: self.is_compiling,
fid,
source: IncrFileQuery::with_context(None),
buffer: FileQuery::default(),
});
if self.is_compiling && !entry.touched_by_compile {
// We put the mutation behind the if statement to avoid
// unnecessary writes to the cache.
entry.touched_by_compile = true;
}
entry
})
}
cache_entry
.map(|e| SourceCache {
last_accessed_rev: self.revision.max(e.last_accessed_rev),
fid,
source: IncrFileQuery::with_context(
e.source
.get_uninitialized()
.cloned()
.transpose()
.ok()
.flatten(),
),
buffer: FileQuery::default(),
})
.unwrap_or_else(|| SourceCache {
last_accessed_rev: self.revision,
fid,
source: IncrFileQuery::with_context(None),
buffer: FileQuery::default(),
})
}))
pub(crate) fn take_state(&mut self) -> SourceDb {
let slots = std::mem::take(&mut self.slots);
SourceDb {
is_compiling: self.is_compiling,
slots,
}
}
}

310
crates/tinymist-world/src/world.rs
View file

@ -6,10 +6,10 @@ use std::{
};
use chrono::{DateTime, Datelike, Local};
use parking_lot::RwLock;
use tinymist_std::error::prelude::*;
use tinymist_vfs::{notify::FilesystemEvent, PathResolution, Vfs, WorkspaceResolver};
use tinymist_vfs::{FsProvider, TypstFileId};
use tinymist_vfs::{
FsProvider, PathResolution, RevisingVfs, SourceCache, TypstFileId, Vfs, WorkspaceResolver,
};
use typst::{
diag::{eco_format, At, EcoString, FileError, FileResult, SourceResult},
foundations::{Bytes, Datetime, Dict},
@ -19,79 +19,21 @@ use typst::{
Library, World,
};
use crate::source::{SharedState, SourceCache, SourceDb};
use crate::{
entry::{EntryManager, EntryReader, EntryState, DETACHED_ENTRY},
font::FontResolver,
package::{PackageRegistry, PackageSpec},
parser::{
get_semantic_tokens_full, get_semantic_tokens_legend, OffsetEncoding, SemanticToken,
SemanticTokensLegend,
},
CompilerFeat, ShadowApi, WorldDeps,
use crate::parser::{
get_semantic_tokens_full, get_semantic_tokens_legend, OffsetEncoding, SemanticToken,
SemanticTokensLegend,
};
use crate::{
package::{PackageRegistry, PackageSpec},
source::SourceDb,
};
// use crate::source::{SharedState, SourceCache, SourceDb};
use crate::entry::{EntryManager, EntryReader, EntryState, DETACHED_ENTRY};
use crate::{font::FontResolver, CompilerFeat, ShadowApi, WorldDeps};
type CodespanResult<T> = Result<T, CodespanError>;
type CodespanError = codespan_reporting::files::Error;
pub struct Revising<'a, T> {
pub revision: NonZeroUsize,
pub inner: &'a mut T,
}
impl<T> std::ops::Deref for Revising<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.inner
}
}
impl<T> std::ops::DerefMut for Revising<'_, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.inner
}
}
impl<F: CompilerFeat> Revising<'_, CompilerUniverse<F>> {
pub fn vfs(&mut self) -> &mut Vfs<F::AccessModel> {
&mut self.inner.vfs
}
/// Let the vfs notify the access model with a filesystem event.
///
/// See `reflexo_vfs::NotifyAccessModel` for more information.
pub fn notify_fs_event(&mut self, event: FilesystemEvent) {
self.inner.vfs.notify_fs_event(event);
}
pub fn reset_shadow(&mut self) {
self.inner.vfs.reset_shadow()
}
pub fn map_shadow(&mut self, path: &Path, content: Bytes) -> FileResult<()> {
self.inner.vfs.map_shadow(path, content)
}
pub fn unmap_shadow(&mut self, path: &Path) -> FileResult<()> {
self.inner.vfs.remove_shadow(path);
Ok(())
}
/// Set the inputs for the compiler.
pub fn set_inputs(&mut self, inputs: Arc<LazyHash<Dict>>) {
self.inner.inputs = inputs;
}
pub fn set_entry_file(&mut self, entry_file: Arc<Path>) -> SourceResult<()> {
self.inner.set_entry_file_(entry_file)
}
pub fn mutate_entry(&mut self, state: EntryState) -> SourceResult<EntryState> {
self.inner.mutate_entry_(state)
}
}
/// A universe that provides access to the operating system.
///
/// Use [`CompilerUniverse::new`] to create a new universe.
@ -111,10 +53,8 @@ pub struct CompilerUniverse<F: CompilerFeat> {
/// Provides path-based data access for typst compiler.
vfs: Vfs<F::AccessModel>,
/// The current revision of the source database.
pub revision: RwLock<NonZeroUsize>,
/// Shared state for source cache.
pub shared: Arc<RwLock<SharedState<SourceCache>>>,
/// The current revision of the universe.
pub revision: NonZeroUsize,
}
/// Creates, snapshots, and manages the compiler universe.
@ -136,8 +76,7 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
entry,
inputs: inputs.unwrap_or_default(),
revision: RwLock::new(NonZeroUsize::new(1).expect("initial revision is 1")),
shared: Arc::new(RwLock::new(SharedState::default())),
revision: NonZeroUsize::new(1).expect("initial revision is 1"),
font_resolver,
registry,
@ -160,8 +99,6 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
}
pub fn snapshot_with(&self, mutant: Option<TaskInputs>) -> CompilerWorld<F> {
let rev_lock = self.revision.read();
let w = CompilerWorld {
entry: self.entry.clone(),
inputs: self.inputs.clone(),
@ -169,9 +106,9 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
font_resolver: self.font_resolver.clone(),
registry: self.registry.clone(),
vfs: self.vfs.snapshot(),
revision: self.revision,
source_db: SourceDb {
revision: *rev_lock,
shared: self.shared.clone(),
is_compiling: true,
slots: Default::default(),
},
now: OnceLock::new(),
@ -181,19 +118,18 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
}
/// Increment revision with actions.
pub fn increment_revision<T>(&mut self, f: impl FnOnce(&mut Revising<Self>) -> T) -> T {
let rev_lock = self.revision.get_mut();
*rev_lock = rev_lock.checked_add(1).unwrap();
let revision = *rev_lock;
f(&mut Revising {
pub fn increment_revision<T>(&mut self, f: impl FnOnce(&mut RevisingUniverse<F>) -> T) -> T {
f(&mut RevisingUniverse {
vfs_revision: self.vfs.revision(),
font_revision: self.font_resolver.revision(),
registry_revision: self.registry.revision(),
view_changed: false,
inner: self,
revision,
})
}
/// Mutate the entry state and return the old state.
fn mutate_entry_(&mut self, mut state: EntryState) -> SourceResult<EntryState> {
self.reset();
std::mem::swap(&mut self.entry, &mut state);
Ok(state)
}
@ -216,10 +152,16 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
impl<F: CompilerFeat> CompilerUniverse<F> {
/// Reset the world for a new lifecycle (of garbage collection).
pub fn reset(&mut self) {
self.vfs.reset();
self.vfs.reset_all();
// todo: shared state
}
/// Clear the vfs cache that is not touched for a long time.
pub fn evict(&mut self, vfs_threshold: usize) {
self.vfs.reset_access_model();
self.vfs.evict(vfs_threshold);
}
/// Resolve the real path for a file id.
pub fn path_for_id(&self, id: FileId) -> Result<PathResolution, FileError> {
self.vfs.file_path(id)
@ -269,7 +211,7 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
impl<F: CompilerFeat> ShadowApi for CompilerUniverse<F> {
#[inline]
fn reset_shadow(&mut self) {
self.increment_revision(|this| this.vfs.reset_shadow())
self.increment_revision(|this| this.vfs.revise().reset_shadow())
}
fn shadow_paths(&self) -> Vec<Arc<Path>> {
@ -282,20 +224,17 @@ impl<F: CompilerFeat> ShadowApi for CompilerUniverse<F> {
#[inline]
fn map_shadow(&mut self, path: &Path, content: Bytes) -> FileResult<()> {
self.increment_revision(|this| this.vfs().map_shadow(path, content))
self.increment_revision(|this| this.vfs().map_shadow(path, Ok(content).into()))
}
#[inline]
fn unmap_shadow(&mut self, path: &Path) -> FileResult<()> {
self.increment_revision(|this| {
this.vfs().remove_shadow(path);
Ok(())
})
self.increment_revision(|this| this.vfs().unmap_shadow(path))
}
#[inline]
fn map_shadow_by_id(&mut self, file_id: FileId, content: Bytes) -> FileResult<()> {
self.increment_revision(|this| this.vfs().map_shadow_by_id(file_id, content))
self.increment_revision(|this| this.vfs().map_shadow_by_id(file_id, Ok(content).into()))
}
#[inline]
@ -314,16 +253,116 @@ impl<F: CompilerFeat> EntryReader for CompilerUniverse<F> {
}
impl<F: CompilerFeat> EntryManager for CompilerUniverse<F> {
fn reset(&mut self) -> SourceResult<()> {
self.reset();
Ok(())
}
fn mutate_entry(&mut self, state: EntryState) -> SourceResult<EntryState> {
self.increment_revision(|this| this.mutate_entry_(state))
}
}
pub struct RevisingUniverse<'a, F: CompilerFeat> {
view_changed: bool,
vfs_revision: NonZeroUsize,
font_revision: Option<NonZeroUsize>,
registry_revision: Option<NonZeroUsize>,
pub inner: &'a mut CompilerUniverse<F>,
}
impl<F: CompilerFeat> std::ops::Deref for RevisingUniverse<'_, F> {
type Target = CompilerUniverse<F>;
fn deref(&self) -> &Self::Target {
self.inner
}
}
impl<F: CompilerFeat> std::ops::DerefMut for RevisingUniverse<'_, F> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.inner
}
}
impl<F: CompilerFeat> Drop for RevisingUniverse<'_, F> {
fn drop(&mut self) {
let mut view_changed = self.view_changed;
// If the revision is none, it means the fonts should be viewed as
// changed unconditionally.
if self.font_changed() {
view_changed = true;
}
// If the revision is none, it means the packages should be viewed as
// changed unconditionally.
if self.registry_changed() {
view_changed = true;
// The registry has changed affects the vfs cache.
self.vfs().reset_cache();
}
let view_changed = view_changed || self.vfs_changed();
if view_changed {
self.vfs.reset_access_model();
let revision = &mut self.revision;
*revision = revision.checked_add(1).unwrap();
}
}
}
impl<F: CompilerFeat> RevisingUniverse<'_, F> {
pub fn vfs(&mut self) -> RevisingVfs<'_, F::AccessModel> {
self.vfs.revise()
}
pub fn set_fonts(&mut self, fonts: Arc<F::FontResolver>) {
self.inner.font_resolver = fonts;
}
pub fn set_package(&mut self, packages: Arc<F::Registry>) {
self.inner.registry = packages;
}
/// Set the inputs for the compiler.
pub fn set_inputs(&mut self, inputs: Arc<LazyHash<Dict>>) {
self.view_changed = true;
self.inner.inputs = inputs;
}
pub fn set_entry_file(&mut self, entry_file: Arc<Path>) -> SourceResult<()> {
self.view_changed = true;
self.inner.set_entry_file_(entry_file)
}
pub fn mutate_entry(&mut self, state: EntryState) -> SourceResult<EntryState> {
self.view_changed = true;
// Resets the cache if the workspace root has changed.
let root_changed = self.inner.entry.workspace_root() == state.workspace_root();
if root_changed {
self.vfs().reset_cache();
}
self.inner.mutate_entry_(state)
}
pub fn flush(&mut self) {
self.view_changed = true;
}
pub fn font_changed(&self) -> bool {
is_revision_changed(self.font_revision, self.font_resolver.revision())
}
pub fn registry_changed(&self) -> bool {
is_revision_changed(self.registry_revision, self.registry.revision())
}
pub fn vfs_changed(&self) -> bool {
self.vfs_revision != self.vfs.revision()
}
}
fn is_revision_changed(a: Option<NonZeroUsize>, b: Option<NonZeroUsize>) -> bool {
a.is_none() || b.is_none() || a != b
}
pub struct CompilerWorld<F: CompilerFeat> {
/// State for the *root & entry* of compilation.
/// The world forbids direct access to files outside this directory.
@ -340,8 +379,9 @@ pub struct CompilerWorld<F: CompilerFeat> {
/// Provides path-based data access for typst compiler.
vfs: Vfs<F::AccessModel>,
revision: NonZeroUsize,
/// Provides source database for typst compiler.
pub source_db: SourceDb,
source_db: SourceDb,
/// The current datetime if requested. This is stored here to ensure it is
/// always the same within one compilation. Reset between compilations.
now: OnceLock<DateTime<Local>>,
@ -353,15 +393,6 @@ impl<F: CompilerFeat> Clone for CompilerWorld<F> {
}
}
impl<F: CompilerFeat> Drop for CompilerWorld<F> {
fn drop(&mut self) {
let state = self.source_db.shared.clone();
let source_state = self.source_db.take_state();
let mut state = state.write();
source_state.commit_impl(&mut state);
}
}
#[derive(Default)]
pub struct TaskInputs {
pub entry: Option<EntryState>,
@ -375,16 +406,44 @@ impl<F: CompilerFeat> CompilerWorld<F> {
let library = mutant.inputs.clone().map(create_library);
CompilerWorld {
let root_changed = if let Some(e) = mutant.entry.as_ref() {
self.entry.workspace_root() != e.workspace_root()
} else {
false
};
let mut world = CompilerWorld {
inputs: mutant.inputs.unwrap_or_else(|| self.inputs.clone()),
library: library.unwrap_or_else(|| self.library.clone()),
entry: mutant.entry.unwrap_or_else(|| self.entry.clone()),
font_resolver: self.font_resolver.clone(),
registry: self.registry.clone(),
vfs: self.vfs.snapshot(),
revision: self.revision,
source_db: self.source_db.clone(),
now: self.now.clone(),
};
if root_changed {
world.vfs.revise().reset_cache();
}
world
}
pub fn take_cache(&mut self) -> SourceCache {
self.vfs.take_state()
}
pub fn take_db(&mut self) -> SourceDb {
self.source_db.take_state()
}
/// Sets flag to indicate whether the compiler is currently compiling.
/// Note: Since `CompilerWorld` can be cloned, you can clone the world and
/// set the flag then to avoid affecting the original world.
pub fn set_is_compiling(&mut self, is_compiling: bool) {
self.source_db.is_compiling = is_compiling;
}
pub fn inputs(&self) -> Arc<LazyHash<Dict>> {
@ -425,7 +484,7 @@ impl<F: CompilerFeat> CompilerWorld<F> {
}
pub fn revision(&self) -> NonZeroUsize {
self.source_db.revision
self.revision
}
}
@ -442,39 +501,44 @@ impl<F: CompilerFeat> ShadowApi for CompilerWorld<F> {
#[inline]
fn reset_shadow(&mut self) {
self.vfs.reset_shadow()
self.vfs.revise().reset_shadow()
}
#[inline]
fn map_shadow(&mut self, path: &Path, content: Bytes) -> FileResult<()> {
self.vfs.map_shadow(path, content)
self.vfs.revise().map_shadow(path, Ok(content).into())
}
#[inline]
fn unmap_shadow(&mut self, path: &Path) -> FileResult<()> {
self.vfs.remove_shadow(path);
Ok(())
self.vfs.revise().unmap_shadow(path)
}
#[inline]
fn map_shadow_by_id(&mut self, file_id: TypstFileId, content: Bytes) -> FileResult<()> {
self.vfs.map_shadow_by_id(file_id, content)
self.vfs
.revise()
.map_shadow_by_id(file_id, Ok(content).into())
}
#[inline]
fn unmap_shadow_by_id(&mut self, file_id: TypstFileId) -> FileResult<()> {
self.vfs.remove_shadow_by_id(file_id);
self.vfs.revise().remove_shadow_by_id(file_id);
Ok(())
}
}
impl<F: CompilerFeat> FsProvider for CompilerWorld<F> {
fn file_path(&self, fid: TypstFileId) -> FileResult<PathResolution> {
self.vfs.file_path(fid)
fn file_path(&self, file_id: TypstFileId) -> FileResult<PathResolution> {
self.vfs.file_path(file_id)
}
fn read(&self, fid: TypstFileId) -> FileResult<Bytes> {
self.vfs.read(fid)
fn read(&self, file_id: TypstFileId) -> FileResult<Bytes> {
self.vfs.read(file_id)
}
fn read_source(&self, file_id: TypstFileId) -> FileResult<Source> {
self.vfs.source(file_id)
}
}

2
crates/tinymist/src/actor/mod.rs
View file

@ -8,7 +8,7 @@ pub mod typ_server;
use std::sync::Arc;
use crate::world::vfs::notify::{FileChangeSet, MemoryEvent};
use crate::world::vfs::{notify::MemoryEvent, FileChangeSet};
use crate::world::EntryState;
use reflexo::ImmutPath;
use tinymist_query::analysis::{Analysis, PeriscopeProvider};

36
crates/tinymist/src/actor/typ_server.rs
View file

@ -12,8 +12,8 @@ use reflexo_typst::{
features::WITH_COMPILING_STATUS_FEATURE, typst::prelude::EcoVec, CompileEnv, CompileReport,
Compiler, ConsoleDiagReporter, FeatureSet, GenericExporter, TypstDocument,
};
use tinymist_project::vfs::FsProvider;
use tinymist_project::watch_deps;
use tinymist_project::{vfs::FsProvider, RevisingUniverse};
use tokio::sync::{mpsc, oneshot};
use typst::diag::{SourceDiagnostic, SourceResult};
@ -26,7 +26,6 @@ use crate::world::base::{
CompilerUniverse,
CompilerWorld,
EntryReader,
Revising,
TaskInputs,
WorldDeps,
};
@ -533,7 +532,7 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
fn process_compile(&mut self, artifact: CompiledArtifact<F>, send: impl Fn(CompilerResponse)) {
self.compiling = false;
let world = &artifact.snap.world;
let mut world = artifact.snap.world;
let compiled_revision = world.revision().get();
if self.committed_revision >= compiled_revision {
return;
@ -559,7 +558,7 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
)));
// Trigger an evict task.
self.cache.evict();
self.cache.evict(world.revision(), world.take_cache());
}
/// Process some interrupt. Return whether it needs compilation.
@ -578,7 +577,7 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
if self
.watch_snap
.get()
.is_some_and(|e| e.world.revision() < *self.verse.revision.read())
.is_some_and(|e| e.world.revision() < self.verse.revision)
{
self.watch_snap = OnceLock::new();
}
@ -613,7 +612,7 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
if self.suspended {
log::info!("CompileServerActor: removing diag");
self.compile_handle
.status(self.verse.revision.get_mut().get(), CompileReport::Suspend);
.status(self.verse.revision.get(), CompileReport::Suspend);
}
// Reset the watch state and document state.
@ -682,7 +681,7 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
// Actual a delayed memory event.
reason = reason_by_mem();
}
verse.notify_fs_event(event)
verse.vfs().notify_fs_event(event)
});
reason
@ -699,7 +698,7 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
/// Apply delayed memory changes to underlying compiler.
fn apply_delayed_memory_changes(
verse: &mut Revising<CompilerUniverse<F>>,
verse: &mut RevisingUniverse<F>,
dirty_shadow_logical_tick: &mut usize,
event: &mut FilesystemEvent,
) -> Option<()> {
@ -723,25 +722,18 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
}
/// Apply memory changes to underlying compiler.
fn apply_memory_changes(verse: &mut Revising<CompilerUniverse<F>>, event: MemoryEvent) {
fn apply_memory_changes(verse: &mut RevisingUniverse<F>, event: MemoryEvent) {
let mut vfs = verse.vfs();
if matches!(event, MemoryEvent::Sync(..)) {
verse.reset_shadow();
vfs.reset_shadow();
}
match event {
MemoryEvent::Update(event) | MemoryEvent::Sync(event) => {
for removes in event.removes {
let _ = verse.unmap_shadow(&removes);
for path in event.removes {
let _ = vfs.unmap_shadow(&path);
}
for (p, t) in event.inserts {
let insert_file = match t.content().cloned() {
Ok(content) => content,
Err(err) => {
log::error!("CompileServerActor: read memory file at {p:?}: {err}");
continue;
}
};
let _ = verse.map_shadow(&p, insert_file);
for (path, snap) in event.inserts {
let _ = vfs.map_shadow(&path, snap);
}
}
}

2
crates/tinymist/src/server.rs
View file

@ -4,7 +4,7 @@ use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::world::vfs::notify::{FileChangeSet, MemoryEvent};
use crate::world::vfs::{notify::MemoryEvent, FileChangeSet};
use actor::editor::EditorActor;
use anyhow::anyhow;
use anyhow::Context;

16
crates/tinymist/src/task/cache.rs
View file

@ -1,17 +1,26 @@
//! The actor that handles cache evicting.
use std::sync::{atomic::AtomicUsize, Arc};
use std::{
num::NonZeroUsize,
sync::{atomic::AtomicUsize, Arc},
};
use crate::world::vfs::SourceCache;
use super::{FutureFolder, SyncTaskFactory};
#[derive(Debug, Clone)]
pub struct CacheUserConfig {
pub max_age: usize,
pub vfs_age: usize,
}
impl Default for CacheUserConfig {
fn default() -> Self {
Self { max_age: 30 }
Self {
max_age: 30,
vfs_age: 15,
}
}
}
@ -31,7 +40,7 @@ impl CacheTask {
}
}
pub fn evict(&self) {
pub fn evict(&self, rev: NonZeroUsize, source_cache: SourceCache) {
let revision = self
.revision
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
@ -42,6 +51,7 @@ impl CacheTask {
// Evict compilation cache.
let evict_start = std::time::Instant::now();
comemo::evict(task.max_age);
source_cache.evict(rev, task.vfs_age);
let elapsed = evict_start.elapsed();
log::info!("CacheEvictTask: evict cache in {elapsed:?}");
})

2
crates/tinymist/src/tool/preview.rs
View file

@ -3,7 +3,7 @@
use std::num::NonZeroUsize;
use std::{collections::HashMap, net::SocketAddr, path::Path, sync::Arc};
use crate::world::vfs::notify::{FileChangeSet, MemoryEvent};
use crate::world::vfs::{notify::MemoryEvent, FileChangeSet};
use futures::{SinkExt, StreamExt, TryStreamExt};
use hyper::service::service_fn;
use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, HyperWebsocketStream};

2
crates/typlite/src/lib.rs
View file

@ -454,7 +454,7 @@ impl TypliteWorker {
entry: Some(entry),
inputs,
});
world.source_db.take_state();
world.take_db();
world.map_shadow_by_id(world.main(), main).unwrap();
let document = typst::compile(&world).output;