language-servers/tinymist
1
0
Fork 0
mirror of https://github.com/Myriad-Dreamin/tinymist.git synced 2025-07-24 13:13:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

feat: encode and use workspace information into PackageSpec (#1187)

Browse source 
* feat: remove an unused API

* feat: encode workspace information into `PackageSpec`

* feat: remove unused real_path

* feat: remove unused mtime

* feat: add ResolveAccessModel

* feat: implement id overlay semantics

* feat: remove mtime checking in overlay model

* feat: remove mtime checking in notify model

* feat: format ids

* fix: cases

* feat: resolve root by world

* dev: add untitled root

* fix: warnings

* fix: a wrong usage

* fix: snapshots

* fix: tests
This commit is contained in:
Myriad-Dreamin 2025-01-19 11:51:00 +08:00 • committed by GitHub
parent a25d208124
commit 56714675b7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 835 additions and 774 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/tinymist-project/src/lock.rs
View file

@ -9,7 +9,7 @@ use crate::LspWorld;
/// Make a new project lock updater.
pub fn update_lock(world: &LspWorld) -> Option<ProjectLockUpdater> {
let root = world.workspace_root()?;
let root = world.entry_state().workspace_root()?;
Some(ProjectLockUpdater {
root,
updates: vec![],

43
crates/tinymist-project/src/watch.rs
View file

@ -18,17 +18,14 @@ use typst::diag::{EcoString, FileError, FileResult};
use crate::vfs::{
notify::{FileChangeSet, FileSnapshot, FilesystemEvent, NotifyMessage, UpstreamUpdateEvent},
system::SystemAccessModel,
AccessModel, Bytes,
Bytes, PathAccessModel,
};
use tinymist_std::ImmutPath;
type WatcherPair = (RecommendedWatcher, mpsc::UnboundedReceiver<NotifyEvent>);
type NotifyEvent = notify::Result<notify::Event>;
type FileEntry = (/* key */ ImmutPath, /* value */ FileSnapshot);
type NotifyFilePair = FileResult<(
/* mtime */ tinymist_std::time::Time,
/* content */ Bytes,
)>;
type NotifyFilePair = FileResult</* content */ Bytes>;
/// The state of a watched file.
///
@ -290,10 +287,7 @@ impl NotifyActor {
changeset.may_insert(self.notify_entry_update(path.clone(), Some(meta)));
} else {
let watched = meta.and_then(|meta| {
let content = self.inner.content(path)?;
Ok((meta.modified().unwrap(), content))
});
let watched = self.inner.content(path);
changeset.inserts.push((path.clone(), watched.into()));
}
}
@ -409,12 +403,8 @@ impl NotifyActor {
return None;
}
// Check meta, path, and content
// Get meta, real path and ignore errors
let mtime = meta.modified().ok()?;
let mut file = self.inner.content(&path).map(|it| (mtime, it));
// Check path and content
let mut file = self.inner.content(&path);
// Check state in fast path: compare state, return None on not sending
// the file change
@ -456,7 +446,7 @@ impl NotifyActor {
}
},
// Compare content for transitional the state
(Some(Ok((prev_tick, prev_content))), Ok((next_tick, next_content))) => {
(Some(Ok(prev_content)), Ok(next_content)) => {
// So far it is accurately no change for the file, skip it
if prev_content == next_content {
return None;
@ -488,27 +478,6 @@ impl NotifyActor {
// Otherwise, we push the diff to the consumer.
WatchState::EmptyOrRemoval { .. } => {}
}
// We have found a change, however, we need to check whether the
// mtime is changed. Generally, the mtime should be changed.
// However, It is common that editor (VSCode) to change the
// mtime after writing
//
// this condition should be never happen, but we still check it
//
// There will be cases that user change content of a file and
// then also modify the mtime of the file, so we need to check
// `next_tick == prev_tick`: Whether mtime is changed.
// `matches!(entry.state, WatchState::Fresh)`: Whether the file
// is fresh. We have not submit the file to the compiler, so
// that is ok.
if next_tick == prev_tick && matches!(entry.state, WatchState::Stable) {
// this is necessary to invalidate our mtime-based cache
*next_tick = prev_tick
.checked_add(std::time::Duration::from_micros(1))
.unwrap();
log::warn!("same content but mtime is different...: {:?} content: prev:{:?} v.s. curr:{:?}", path, prev_content, next_content);
};
}
};

20
crates/tinymist-project/src/world.rs
View file

@ -4,6 +4,7 @@ pub use tinymist_std::error::prelude;
pub use tinymist_world as base;
pub use tinymist_world::args::*;
pub use tinymist_world::config::CompileFontOpts;
use tinymist_world::package::RegistryPathMapper;
pub use tinymist_world::vfs;
pub use tinymist_world::{entry::*, EntryOpts, EntryState};
pub use tinymist_world::{font, package, CompilerUniverse, CompilerWorld, Revising, TaskInputs};
@ -132,15 +133,30 @@ impl LspUniverseBuilder {
font_resolver: Arc<TinymistFontResolver>,
package_registry: HttpRegistry,
) -> ZResult<LspUniverse> {
let registry = Arc::new(package_registry);
let resolver = Arc::new(RegistryPathMapper::new(registry.clone()));
Ok(LspUniverse::new_raw(
entry,
Some(inputs),
Vfs::new(SystemAccessModel {}),
package_registry,
Vfs::new(resolver, SystemAccessModel {}),
registry,
font_resolver,
))
}
/// Resolve fonts from given options.
pub fn only_embedded_fonts() -> ZResult<TinymistFontResolver> {
let mut searcher = SystemFontSearcher::new();
searcher.resolve_opts(CompileFontOpts {
font_profile_cache_path: Default::default(),
font_paths: vec![],
no_system_fonts: true,
with_embedded_fonts: typst_assets::fonts().map(Cow::Borrowed).collect(),
})?;
Ok(searcher.into())
}
/// Resolve fonts from given options.
pub fn resolve_fonts(args: CompileFontArgs) -> ZResult<TinymistFontResolver> {
let mut searcher = SystemFontSearcher::new();

2
crates/tinymist-query/Cargo.toml
View file

@ -59,7 +59,7 @@ hashbrown.workspace = true
triomphe.workspace = true
base64.workspace = true
typlite.workspace = true
tinymist-world.workspace = true
tinymist-world = { workspace = true, features = ["no-content-hint"] }
tinymist-project.workspace = true
tinymist-analysis.workspace = true
tinymist-derive.workspace = true

38
crates/tinymist-query/src/analysis.rs
View file

@ -24,7 +24,8 @@ pub mod signature;
pub use signature::*;
pub mod semantic_tokens;
pub use semantic_tokens::*;
use typst::syntax::{Source, VirtualPath};
use tinymist_world::vfs::WorkspaceResolver;
use typst::syntax::Source;
use typst::World;
mod post_tyck;
mod tyck;
@ -40,12 +41,11 @@ pub use global::*;
use ecow::eco_format;
use lsp_types::Url;
use tinymist_world::EntryReader;
use typst::diag::{FileError, FileResult};
use typst::foundations::{Func, Value};
use typst::syntax::FileId;
use crate::path_to_url;
use crate::path_res_to_url;
pub(crate) trait ToFunc {
fn to_func(&self) -> Option<Func>;
@ -76,16 +76,13 @@ pub trait LspWorldExt {
impl LspWorldExt for tinymist_project::LspWorld {
fn file_id_by_path(&self, path: &Path) -> FileResult<FileId> {
// todo: source in packages
let root = self.workspace_root().ok_or_else(|| {
let reason = eco_format!("workspace root not found");
FileError::Other(Some(reason))
})?;
let relative_path = path.strip_prefix(&root).map_err(|_| {
let reason = eco_format!("access denied, path: {path:?}, root: {root:?}");
FileError::Other(Some(reason))
})?;
Ok(FileId::new(None, VirtualPath::new(relative_path)))
match self.id_for_path(path) {
Some(id) => Ok(id),
None => WorkspaceResolver::file_with_parent_root(path).ok_or_else(|| {
let reason = eco_format!("invalid path: {path:?}");
FileError::Other(Some(reason))
}),
}
}
fn source_by_path(&self, path: &Path) -> FileResult<Source> {
@ -94,10 +91,10 @@ impl LspWorldExt for tinymist_project::LspWorld {
}
fn uri_for_id(&self, fid: FileId) -> Result<Url, FileError> {
self.path_for_id(fid).and_then(|path| {
path_to_url(&path)
.map_err(|err| FileError::Other(Some(eco_format!("convert to url: {err:?}"))))
})
let res = path_res_to_url(self.path_for_id(fid)?);
log::info!("uri_for_id: {fid:?} -> {res:?}");
res.map_err(|err| FileError::Other(Some(eco_format!("convert to url: {err:?}"))))
}
}
@ -133,6 +130,7 @@ mod matcher_tests {
mod expr_tests {
use tinymist_std::path::unix_slash;
use tinymist_world::vfs::WorkspaceResolver;
use typst::syntax::Source;
use crate::syntax::{Expr, RefExpr};
@ -150,8 +148,10 @@ mod expr_tests {
let fid = if let Some(fid) = decl.file_id() {
let vpath = fid.vpath().as_rooted_path();
match fid.package() {
Some(package) => format!(" in {package:?}{}", unix_slash(vpath)),
None => format!(" in {}", unix_slash(vpath)),
Some(package) if WorkspaceResolver::is_package_file(fid) => {
format!(" in {package:?}{}", unix_slash(vpath))
}
Some(_) | None => format!(" in {}", unix_slash(vpath)),
}
} else {
"".to_string()

4
crates/tinymist-query/src/analysis/completion/path.rs
View file

@ -1,10 +1,12 @@
//! Completion of paths (string literal).
use tinymist_world::vfs::WorkspaceResolver;
use super::*;
impl CompletionPair<'_, '_, '_> {
pub fn complete_path(&mut self, preference: &PathPreference) -> Option<Vec<CompletionItem>> {
let id = self.cursor.source.id();
if id.package().is_some() {
if WorkspaceResolver::is_package_file(id) {
return None;
}

27
crates/tinymist-query/src/analysis/global.rs
View file

@ -11,6 +11,7 @@ use rustc_hash::FxHashMap;
use tinymist_project::LspWorld;
use tinymist_std::debug_loc::DataSource;
use tinymist_std::hash::{hash128, FxDashMap};
use tinymist_world::vfs::{PathResolution, WorkspaceResolver};
use tinymist_world::{EntryReader, WorldDeps, DETACHED_ENTRY};
use typst::diag::{eco_format, At, FileError, FileResult, SourceResult, StrResult};
use typst::engine::{Route, Sink, Traced};
@ -315,9 +316,9 @@ impl LocalContext {
self.caches
.completion_files
.get_or_init(|| {
if let Some(root) = self.world.workspace_root() {
if let Some(root) = self.world.entry_state().workspace_root() {
scan_workspace_files(&root, PathPreference::Special.ext_matcher(), |path| {
TypstFileId::new(None, VirtualPath::new(path))
WorkspaceResolver::workspace_file(Some(&root), VirtualPath::new(path))
})
} else {
vec![]
@ -357,7 +358,7 @@ impl LocalContext {
}
/// Get all depended files in the workspace, inclusively.
pub fn depended_source_files(&self) -> Vec<TypstFileId> {
pub fn depended_source_files(&self) -> EcoVec<TypstFileId> {
let mut ids = self.depended_files();
let preference = PathPreference::Source {
allow_package: false,
@ -368,22 +369,10 @@ impl LocalContext {
/// Get all depended file ids of a compilation, inclusively.
/// Note: must be called after compilation.
pub fn depended_files(&self) -> Vec<TypstFileId> {
let mut ids = vec![];
for dep in self.depended_paths() {
if let Ok(ref_fid) = self.file_id_by_path(&dep) {
ids.push(ref_fid);
}
}
ids
}
/// Get depended paths of a compilation.
/// Note: must be called after compilation.
pub(crate) fn depended_paths(&self) -> EcoVec<tinymist_std::ImmutPath> {
pub fn depended_files(&self) -> EcoVec<TypstFileId> {
let mut deps = EcoVec::new();
self.world.iter_dependencies(&mut |path| {
deps.push(path);
self.world.iter_dependencies(&mut |file_id| {
deps.push(file_id);
});
deps
@ -585,7 +574,7 @@ impl SharedContext {
}
/// Resolve the real path for a file id.
pub fn path_for_id(&self, id: TypstFileId) -> Result<PathBuf, FileError> {
pub fn path_for_id(&self, id: TypstFileId) -> Result<PathResolution, FileError> {
self.world.path_for_id(id)
}

3
crates/tinymist-query/src/analysis/link_expr.rs
View file

@ -40,9 +40,8 @@ impl LinkTarget {
LinkTarget::Url(url) => Some(url.as_ref().clone()),
LinkTarget::Path(id, path) => {
// Avoid creating new ids here.
let base = id.vpath().join(path.as_str());
let root = ctx.path_for_id(id.join("/")).ok()?;
crate::path_to_url(&base.resolve(&root)?).ok()
crate::path_res_to_url(root.join(path).ok()?).ok()
}
}
}

2
crates/tinymist-query/src/analysis/prelude.rs
View file

@ -2,7 +2,7 @@ pub use core::fmt;
pub use std::collections::{BTreeMap, HashMap};
pub use std::hash::{Hash, Hasher};
pub use std::ops::Range;
pub use std::path::{Path, PathBuf};
pub use std::path::Path;
pub use std::sync::{Arc, LazyLock};
pub use comemo::Track;

2
crates/tinymist-query/src/completion.rs
View file

@ -169,7 +169,7 @@ mod tests {
let mut results = vec![];
for s in rng.clone() {
let request = CompletionRequest {
path: ctx.path_for_id(id).unwrap(),
path: ctx.path_for_id(id).unwrap().as_path().to_owned(),
position: ctx.to_lsp_pos(s, &source),
explicit: false,
trigger_character,

24
crates/tinymist-query/src/diagnostics.rs
View file

@ -1,5 +1,4 @@
use tinymist_project::LspWorld;
use tinymist_world::EntryReader;
use typst::syntax::Span;
use crate::{prelude::*, LspWorldExt};
@ -54,19 +53,10 @@ fn convert_diagnostic(
ctx: &LocalDiagContext,
typst_diagnostic: &TypstDiagnostic,
) -> anyhow::Result<(Url, Diagnostic)> {
let uri;
let lsp_range;
if let Some((id, span)) = diagnostic_span_id(typst_diagnostic) {
uri = ctx.uri_for_id(id)?;
let source = ctx.source(id)?;
lsp_range = diagnostic_range(&source, span, ctx.position_encoding);
} else {
let root = ctx
.workspace_root()
.ok_or_else(|| anyhow::anyhow!("no workspace root"))?;
uri = path_to_url(&root)?;
lsp_range = LspRange::default();
};
let (id, span) = diagnostic_span_id(ctx, typst_diagnostic);
let uri = ctx.uri_for_id(id)?;
let source = ctx.source(id)?;
let lsp_range = diagnostic_range(&source, span, ctx.position_encoding);
let lsp_severity = diagnostic_severity(typst_diagnostic.severity);
@ -131,10 +121,14 @@ fn diagnostic_related_information(
Ok(tracepoints)
}
fn diagnostic_span_id(typst_diagnostic: &TypstDiagnostic) -> Option<(TypstFileId, Span)> {
fn diagnostic_span_id(
ctx: &LocalDiagContext,
typst_diagnostic: &TypstDiagnostic,
) -> (TypstFileId, Span) {
iter::once(typst_diagnostic.span)
.chain(typst_diagnostic.trace.iter().map(|trace| trace.span))
.find_map(|span| Some((span.id()?, span)))
.unwrap_or_else(|| (ctx.main(), Span::detached()))
}
fn diagnostic_range(

22
crates/tinymist-query/src/docs/convert.rs
View file

@ -1,37 +1,29 @@
use std::path::Path;
use std::sync::{Arc, LazyLock};
use ecow::{eco_format, EcoString};
use parking_lot::Mutex;
use tinymist_world::{EntryState, ShadowApi, TaskInputs};
use tinymist_world::{EntryReader, ShadowApi, TaskInputs};
use typlite::scopes::Scopes;
use typlite::value::Value;
use typlite::TypliteFeat;
use typst::diag::StrResult;
use typst::foundations::Bytes;
use typst::{
diag::StrResult,
syntax::{FileId, VirtualPath},
};
use typst::World;
use crate::analysis::SharedContext;
// Unfortunately, we have only 65536 possible file ids and we cannot revoke
// them. So we share a global file id for all docs conversion.
static DOCS_CONVERT_ID: LazyLock<Mutex<FileId>> =
LazyLock::new(|| Mutex::new(FileId::new(None, VirtualPath::new("__tinymist_docs__.typ"))));
pub(crate) fn convert_docs(ctx: &SharedContext, content: &str) -> StrResult<EcoString> {
static DOCS_LIB: LazyLock<Arc<Scopes<Value>>> =
LazyLock::new(|| Arc::new(typlite::library::docstring_lib()));
let conv_id = DOCS_CONVERT_ID.lock();
let entry = EntryState::new_rootless(conv_id.vpath().as_rooted_path().into()).unwrap();
let entry = entry.select_in_workspace(*conv_id);
let entry = ctx.world.entry_state();
let entry = entry.select_in_workspace(Path::new("__tinymist_docs__.typ"));
let mut w = ctx.world.task(TaskInputs {
entry: Some(entry),
inputs: None,
});
w.map_shadow_by_id(*conv_id, Bytes::from(content.as_bytes().to_owned()))?;
w.map_shadow_by_id(w.main(), Bytes::from(content.as_bytes().to_owned()))?;
// todo: bad performance
w.source_db.take_state();

2
crates/tinymist-query/src/document_metrics.rs
View file

@ -215,7 +215,7 @@ impl DocumentMetricsWorker<'_> {
let column = source.byte_to_column(byte_index)?;
let filepath = self.ctx.path_for_id(file_id).ok()?;
let filepath_str = filepath.to_string_lossy().to_string();
let filepath_str = filepath.as_path().display().to_string();
Some((filepath_str, line as u32 + 1, column as u32 + 1))
}

25
crates/tinymist-query/src/entry.rs
View file

@ -1,7 +1,7 @@
use anyhow::bail;
use tinymist_std::ImmutPath;
use tinymist_world::EntryState;
use typst::syntax::{FileId, VirtualPath};
use typst::syntax::VirtualPath;
/// Entry resolver
#[derive(Debug, Default, Clone)]
@ -60,14 +60,14 @@ impl EntryResolver {
(Some(entry), Some(root)) => match entry.strip_prefix(&root) {
Ok(stripped) => Some(EntryState::new_rooted(
root,
Some(FileId::new(None, VirtualPath::new(stripped))),
Some(VirtualPath::new(stripped)),
)),
Err(err) => {
log::info!("Entry is not in root directory: err {err:?}: entry: {entry:?}, root: {root:?}");
EntryState::new_rootless(entry)
EntryState::new_rooted_by_parent(entry)
}
},
(Some(entry), None) => EntryState::new_rootless(entry),
(Some(entry), None) => EntryState::new_rooted_by_parent(entry),
(None, Some(root)) => Some(EntryState::new_workspace(root)),
(None, None) => None,
};
@ -106,6 +106,8 @@ impl EntryResolver {
#[cfg(test)]
#[cfg(any(windows, unix, target_os = "macos"))]
mod entry_tests {
use tinymist_world::vfs::WorkspaceResolver;
use super::*;
use std::path::Path;
@ -127,7 +129,10 @@ mod entry_tests {
assert_eq!(entry.root(), Some(ImmutPath::from(root_path)));
assert_eq!(
entry.main(),
Some(FileId::new(None, VirtualPath::new("main.typ")))
Some(WorkspaceResolver::workspace_file(
entry.root().as_ref(),
VirtualPath::new("main.typ")
))
);
}
@ -152,7 +157,10 @@ mod entry_tests {
assert_eq!(entry.root(), Some(ImmutPath::from(root_path)));
assert_eq!(
entry.main(),
Some(FileId::new(None, VirtualPath::new("main.typ")))
Some(WorkspaceResolver::workspace_file(
entry.root().as_ref(),
VirtualPath::new("main.typ")
))
);
}
@ -166,7 +174,10 @@ mod entry_tests {
assert_eq!(entry.root(), Some(ImmutPath::from(root2_path)));
assert_eq!(
entry.main(),
Some(FileId::new(None, VirtualPath::new("main.typ")))
Some(WorkspaceResolver::workspace_file(
entry.root().as_ref(),
VirtualPath::new("main.typ")
))
);
}
}

16
crates/tinymist-query/src/lsp_typst_boundary.rs
View file

@ -3,6 +3,7 @@
use std::cmp::Ordering;
use tinymist_std::path::PathClean;
use tinymist_world::vfs::PathResolution;
use typst::syntax::Source;
use crate::prelude::*;
@ -42,6 +43,11 @@ impl From<PositionEncoding> for lsp_types::PositionEncodingKind {
const UNTITLED_ROOT: &str = "/untitled";
static EMPTY_URL: LazyLock<Url> = LazyLock::new(|| Url::parse("file://").unwrap());
/// Convert a path to a URL.
pub fn untitled_url(path: &Path) -> anyhow::Result<Url> {
Ok(Url::parse(&format!("untitled:{}", path.display()))?)
}
/// Convert a path to a URL.
pub fn path_to_url(path: &Path) -> anyhow::Result<Url> {
if let Ok(untitled) = path.strip_prefix(UNTITLED_ROOT) {
@ -50,7 +56,7 @@ pub fn path_to_url(path: &Path) -> anyhow::Result<Url> {
return Ok(EMPTY_URL.clone());
}
return Ok(Url::parse(&format!("untitled:{}", untitled.display()))?);
return untitled_url(untitled);
}
Url::from_file_path(path).or_else(|never| {
@ -60,6 +66,14 @@ pub fn path_to_url(path: &Path) -> anyhow::Result<Url> {
})
}
/// Convert a path resolution to a URL.
pub fn path_res_to_url(path: PathResolution) -> anyhow::Result<Url> {
match path {
PathResolution::Rootless(path) => untitled_url(path.as_rooted_path()),
PathResolution::Resolved(path) => path_to_url(&path),
}
}
/// Convert a URL to a path.
pub fn url_to_path(uri: Url) -> PathBuf {
if uri.scheme() == "file" {

4
crates/tinymist-query/src/prepare_rename.rs
View file

@ -1,3 +1,5 @@
use tinymist_world::vfs::WorkspaceResolver;
use crate::{
analysis::Definition,
prelude::*,
@ -65,7 +67,7 @@ pub(crate) fn prepare_renaming(
let name = def.name().clone();
let (def_fid, _def_range) = def.location(ctx.shared()).clone()?;
if def_fid.package().is_some() {
if WorkspaceResolver::is_package_file(def_fid) {
crate::log_debug_ct!(
"prepare_rename: {name} is in a package {pkg:?}",
pkg = def_fid.package(),

10
crates/tinymist-query/src/references.rs
View file

@ -80,15 +80,7 @@ struct ReferencesWorker<'a> {
impl ReferencesWorker<'_> {
fn label_root(mut self) -> Option<Vec<LspLocation>> {
let mut ids = vec![];
for dep in self.ctx.ctx.depended_paths() {
if let Ok(ref_fid) = self.ctx.ctx.file_id_by_path(&dep) {
ids.push(ref_fid);
}
}
for ref_fid in ids {
for ref_fid in self.ctx.ctx.depended_files() {
self.file(ref_fid)?;
}

3
crates/tinymist-query/src/rename.rs
View file

@ -59,7 +59,8 @@ impl StatefulRequest for RenameRequest {
};
let def_fid = def.location(ctx.shared())?.0;
let old_path = ctx.path_for_id(def_fid).ok()?;
// todo: rename in untitled files
let old_path = ctx.path_for_id(def_fid).ok()?.to_err().ok()?;
let rename_loc = Path::new(ref_path_str.as_str());
let diff = pathdiff::diff_paths(Path::new(&new_path_str), rename_loc)?;

6
crates/tinymist-query/src/symbol.rs
View file

@ -36,11 +36,11 @@ impl SemanticRequest for SymbolRequest {
let mut symbols = vec![];
for path in ctx.depended_paths() {
let Ok(source) = ctx.source_by_path(&path) else {
for id in ctx.depended_files() {
let Ok(source) = ctx.source_by_id(id) else {
continue;
};
let uri = path_to_url(&path).unwrap();
let uri = ctx.uri_for_id(id).unwrap();
let res = get_lexical_hierarchy(&source, LexicalScopeKind::Symbol).map(|symbols| {
filter_document_symbols(
&symbols,

35
crates/tinymist-query/src/tests.rs
View file

@ -12,18 +12,18 @@ use once_cell::sync::Lazy;
use serde_json::{ser::PrettyFormatter, Serializer, Value};
use tinymist_project::CompileFontArgs;
use tinymist_world::package::PackageSpec;
use tinymist_world::vfs::WorkspaceResolver;
use tinymist_world::EntryState;
use tinymist_world::TaskInputs;
use tinymist_world::{EntryManager, EntryReader, ShadowApi};
use typst::foundations::Bytes;
use typst::syntax::ast::{self, AstNode};
use typst::syntax::{FileId as TypstFileId, LinkedNode, Source, SyntaxKind, VirtualPath};
use typst::syntax::{LinkedNode, Source, SyntaxKind, VirtualPath};
pub use insta::assert_snapshot;
pub use serde::Serialize;
pub use serde_json::json;
pub use tinymist_project::{LspUniverse, LspUniverseBuilder};
use typst::World;
use typst_shim::syntax::LinkedNodeExt;
use crate::syntax::find_module_level_docs;
@ -57,11 +57,16 @@ pub fn run_with_ctx<T>(
path: PathBuf,
f: &impl Fn(&mut LocalContext, PathBuf) -> T,
) -> T {
let root = verse.workspace_root().unwrap();
let root = verse.entry_state().workspace_root().unwrap();
let paths = verse
.shadow_paths()
.into_iter()
.map(|path| TypstFileId::new(None, VirtualPath::new(path.strip_prefix(&root).unwrap())))
.map(|path| {
WorkspaceResolver::workspace_file(
Some(&root),
VirtualPath::new(path.strip_prefix(&root).unwrap()),
)
})
.collect::<Vec<_>>();
let world = verse.snapshot();
@ -115,20 +120,19 @@ pub fn compile_doc_for_test(
ctx: &mut LocalContext,
properties: &HashMap<&str, &str>,
) -> Option<VersionedDocument> {
let main_id = properties.get("compile").and_then(|v| match v.trim() {
"true" => Some(ctx.world.main()),
"false" => None,
let entry = match properties.get("compile")?.trim() {
"true" => ctx.world.entry_state(),
"false" => return None,
path if path.ends_with(".typ") => {
let vp = VirtualPath::new(path);
Some(TypstFileId::new(None, vp))
ctx.world.entry_state().select_in_workspace(Path::new(path))
}
_ => panic!("invalid value for 'compile' property: {v}"),
})?;
v => panic!("invalid value for 'compile' property: {v}"),
};
let mut world = Cow::Borrowed(&ctx.world);
if main_id != ctx.world.main() {
if entry != ctx.world.entry_state() {
world = Cow::Owned(world.task(TaskInputs {
entry: Some(world.entry_state().select_in_workspace(main_id)),
entry: Some(entry),
..Default::default()
}));
}
@ -192,10 +196,7 @@ pub fn run_with_sources<T>(source: &str, f: impl FnOnce(&mut LspUniverse, PathBu
verse
.mutate_entry(EntryState::new_rooted(
root.as_path().into(),
Some(TypstFileId::new(
None,
VirtualPath::new(pw.strip_prefix(root).unwrap()),
)),
Some(VirtualPath::new(pw.strip_prefix(root).unwrap())),
))
.unwrap();
f(&mut verse, pw)

3
crates/tinymist-query/src/workspace_label.rs
View file

@ -25,10 +25,9 @@ impl SemanticRequest for WorkspaceLabelRequest {
let Ok(source) = ctx.source_by_id(fid) else {
continue;
};
let Ok(path) = ctx.path_for_id(fid) else {
let Ok(uri) = ctx.uri_for_id(fid) else {
continue;
};
let uri = path_to_url(&path).unwrap();
let res = get_lexical_hierarchy(&source, LexicalScopeKind::Symbol).map(|hierarchy| {
filter_document_labels(&hierarchy, &source, &uri, ctx.position_encoding())
});

1
crates/tinymist-vfs/Cargo.toml
View file

@ -15,6 +15,7 @@ tinymist-std = { workspace = true, features = ["typst"] }
parking_lot.workspace = true
nohash-hasher.workspace = true
indexmap.workspace = true
comemo.workspace = true
log.workspace = true
rpds = "1"

36
crates/tinymist-vfs/src/browser.rs
View file

@ -1,10 +1,9 @@
use std::path::Path;
use tinymist_std::ImmutPath;
use typst::diag::{FileError, FileResult};
use wasm_bindgen::prelude::*;
use crate::{AccessModel, Bytes, Time};
use crate::{Bytes, PathAccessModel};
/// Provides proxy access model from typst compiler to some JavaScript
/// implementation.
@ -23,24 +22,7 @@ pub struct ProxyAccessModel {
pub read_all_fn: js_sys::Function,
}
impl AccessModel for ProxyAccessModel {
fn mtime(&self, src: &Path) -> FileResult<Time> {
self.mtime_fn
.call1(&self.context, &src.to_string_lossy().as_ref().into())
.map(|v| {
let v = v.as_f64().unwrap();
Time::UNIX_EPOCH + std::time::Duration::from_secs_f64(v)
})
.map_err(|e| {
web_sys::console::error_3(
&"typst_ts::compiler::ProxyAccessModel::mtime failure".into(),
&src.to_string_lossy().as_ref().into(),
&e,
);
FileError::AccessDenied
})
}
impl PathAccessModel for ProxyAccessModel {
fn is_file(&self, src: &Path) -> FileResult<bool> {
self.is_file_fn
.call1(&self.context, &src.to_string_lossy().as_ref().into())
@ -55,20 +37,6 @@ impl AccessModel for ProxyAccessModel {
})
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
self.real_path_fn
.call1(&self.context, &src.to_string_lossy().as_ref().into())
.map(|v| Path::new(&v.as_string().unwrap()).into())
.map_err(|e| {
web_sys::console::error_3(
&"typst_ts::compiler::ProxyAccessModel::real_path failure".into(),
&src.to_string_lossy().as_ref().into(),
&e,
);
FileError::AccessDenied
})
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
let data = self
.read_all_fn

20
crates/tinymist-vfs/src/dummy.rs
View file

@ -1,10 +1,8 @@
use std::path::Path;
use tinymist_std::ImmutPath;
use typst::diag::{FileError, FileResult};
use super::AccessModel;
use crate::{Bytes, Time};
use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
/// Provides dummy access model.
///
@ -15,16 +13,18 @@ use crate::{Bytes, Time};
pub struct DummyAccessModel;
impl AccessModel for DummyAccessModel {
fn mtime(&self, _src: &Path) -> FileResult<Time> {
Ok(Time::UNIX_EPOCH)
}
fn is_file(&self, _src: &Path) -> FileResult<bool> {
fn is_file(&self, _src: TypstFileId) -> FileResult<bool> {
Ok(true)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
Ok(src.into())
fn content(&self, _src: TypstFileId) -> FileResult<Bytes> {
Err(FileError::AccessDenied)
}
}
impl PathAccessModel for DummyAccessModel {
fn is_file(&self, _src: &Path) -> FileResult<bool> {
Ok(true)
}
fn content(&self, _src: &Path) -> FileResult<Bytes> {

202
crates/tinymist-vfs/src/lib.rs
View file

@ -25,40 +25,51 @@ pub mod notify;
/// Provides overlay access model which allows to shadow the underlying access
/// model with memory contents.
pub mod overlay;
/// Provides resolve access model.
pub mod resolve;
/// Provides trace access model which traces the underlying access model.
pub mod trace;
mod utils;
mod path_interner;
mod path_mapper;
use notify::{FilesystemEvent, NotifyAccessModel};
pub use path_mapper::{PathResolution, RootResolver, WorkspaceResolution, WorkspaceResolver};
use resolve::ResolveAccessModel;
pub use typst::foundations::Bytes;
pub use typst::syntax::FileId as TypstFileId;
pub use tinymist_std::time::Time;
pub use tinymist_std::ImmutPath;
pub(crate) use path_interner::PathInterner;
use core::fmt;
use std::{collections::HashMap, hash::Hash, path::Path, sync::Arc};
use std::{path::Path, sync::Arc};
use parking_lot::{Mutex, RwLock};
use tinymist_std::path::PathClean;
use parking_lot::RwLock;
use typst::diag::{FileError, FileResult};
use self::{
notify::{FilesystemEvent, NotifyAccessModel},
overlay::OverlayAccessModel,
};
use self::overlay::OverlayAccessModel;
/// Handle to a file in [`Vfs`]
///
/// Most functions in typst-ts use this when they need to refer to a file.
#[derive(Copy, Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)]
pub struct FileId(pub u32);
pub type FileId = TypstFileId;
/// safe because `FileId` is a new type of `u32`
impl nohash_hasher::IsEnabled for FileId {}
/// A trait for accessing underlying file system.
///
/// This trait is simplified by [`Vfs`] and requires a minimal method set for
/// typst compilation.
pub trait PathAccessModel {
/// Clear the cache of the access model.
///
/// This is called when the vfs is reset. See [`Vfs`]'s reset method for
/// more information.
fn clear(&mut self) {}
/// Return whether a path is corresponding to a file.
fn is_file(&self, src: &Path) -> FileResult<bool>;
/// Return the content of a file entry.
fn content(&self, src: &Path) -> FileResult<Bytes>;
}
/// A trait for accessing underlying file system.
///
@ -70,23 +81,12 @@ pub trait AccessModel {
/// This is called when the vfs is reset. See [`Vfs`]'s reset method for
/// more information.
fn clear(&mut self) {}
/// Return a mtime corresponding to the path.
///
/// Note: vfs won't touch the file entry if mtime is same between vfs reset
/// lifecycles for performance design.
fn mtime(&self, src: &Path) -> FileResult<Time>;
/// Return whether a path is corresponding to a file.
fn is_file(&self, src: &Path) -> FileResult<bool>;
/// Return the real path before creating a vfs file entry.
///
/// Note: vfs will fetch the file entry once if multiple paths shares a same
/// real path.
fn real_path(&self, src: &Path) -> FileResult<ImmutPath>;
fn is_file(&self, src: TypstFileId) -> FileResult<bool>;
/// Return the content of a file entry.
fn content(&self, src: &Path) -> FileResult<Bytes>;
fn content(&self, src: TypstFileId) -> FileResult<Bytes>;
}
#[derive(Clone)]
@ -102,110 +102,60 @@ impl<M> SharedAccessModel<M> {
}
}
impl<M> AccessModel for SharedAccessModel<M>
impl<M> PathAccessModel for SharedAccessModel<M>
where
M: AccessModel,
M: PathAccessModel,
{
fn clear(&mut self) {
self.inner.write().clear();
}
fn mtime(&self, src: &Path) -> FileResult<Time> {
self.inner.read().mtime(src)
}
fn is_file(&self, src: &Path) -> FileResult<bool> {
self.inner.read().is_file(src)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
self.inner.read().real_path(src)
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
self.inner.read().content(src)
}
}
type VfsPathAccessModel<M> = OverlayAccessModel<ImmutPath, NotifyAccessModel<SharedAccessModel<M>>>;
/// we add notify access model here since notify access model doesn't introduce
/// overheads by our observation
type VfsAccessModel<M> = OverlayAccessModel<NotifyAccessModel<SharedAccessModel<M>>>;
type VfsAccessModel<M> = OverlayAccessModel<TypstFileId, ResolveAccessModel<VfsPathAccessModel<M>>>;
pub trait FsProvider {
/// Arbitrary one of file path corresponding to the given `id`.
fn file_path(&self, id: FileId) -> ImmutPath;
fn file_path(&self, id: TypstFileId) -> FileResult<PathResolution>;
fn mtime(&self, id: FileId) -> FileResult<Time>;
fn read(&self, id: TypstFileId) -> FileResult<Bytes>;
fn read(&self, id: FileId) -> FileResult<Bytes>;
fn is_file(&self, id: FileId) -> FileResult<bool>;
fn is_file(&self, id: TypstFileId) -> FileResult<bool>;
}
#[derive(Default)]
struct PathMapper {
/// Map from path to slot index.
///
/// Note: we use a owned [`FileId`] here, which is resultant from
/// [`PathInterner`]
id_cache: RwLock<HashMap<ImmutPath, FileId>>,
/// The path interner for canonical paths.
intern: Mutex<PathInterner<ImmutPath, ()>>,
}
impl PathMapper {
/// Id of the given path if it exists in the `Vfs` and is not deleted.
pub fn file_id(&self, path: &Path) -> FileId {
let quick_id = self.id_cache.read().get(path).copied();
if let Some(id) = quick_id {
return id;
}
let path: ImmutPath = path.clean().as_path().into();
let mut path_interner = self.intern.lock();
let id = path_interner.intern(path.clone(), ()).0;
let mut path2slot = self.id_cache.write();
path2slot.insert(path.clone(), id);
id
}
/// File path corresponding to the given `file_id`.
pub fn file_path(&self, file_id: FileId) -> ImmutPath {
let path_interner = self.intern.lock();
path_interner.lookup(file_id).clone()
}
}
/// Create a new `Vfs` harnessing over the given `access_model` specific for
/// `reflexo_world::CompilerWorld`. With vfs, we can minimize the
/// implementation overhead for [`AccessModel`] trait.
pub struct Vfs<M: AccessModel + Sized> {
paths: Arc<PathMapper>,
pub struct Vfs<M: PathAccessModel + Sized> {
// access_model: TraceAccessModel<VfsAccessModel<M>>,
/// The wrapped access model.
access_model: VfsAccessModel<M>,
}
impl<M: AccessModel + Sized> fmt::Debug for Vfs<M> {
impl<M: PathAccessModel + Sized> fmt::Debug for Vfs<M> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Vfs").finish()
}
}
impl<M: AccessModel + Clone + Sized> Vfs<M> {
impl<M: PathAccessModel + Clone + Sized> Vfs<M> {
pub fn snapshot(&self) -> Self {
Self {
paths: self.paths.clone(),
access_model: self.access_model.clone(),
}
}
}
impl<M: AccessModel + Sized> Vfs<M> {
impl<M: PathAccessModel + Sized> Vfs<M> {
/// Create a new `Vfs` with a given `access_model`.
///
/// Retrieving an [`AccessModel`], it will further wrap the access model
@ -218,18 +168,20 @@ impl<M: AccessModel + Sized> Vfs<M> {
/// the vfs is watching the file system.
///
/// See [`AccessModel`] for more information.
pub fn new(access_model: M) -> Self {
pub fn new(resolver: Arc<dyn RootResolver + Send + Sync>, access_model: M) -> Self {
let access_model = SharedAccessModel::new(access_model);
let access_model = NotifyAccessModel::new(access_model);
let access_model = OverlayAccessModel::new(access_model);
let access_model = ResolveAccessModel {
resolver,
inner: access_model,
};
let access_model = OverlayAccessModel::new(access_model);
// If you want to trace the access model, uncomment the following line
// let access_model = TraceAccessModel::new(access_model);
Self {
paths: Default::default(),
access_model,
}
Self { access_model }
}
/// Reset the source file and path references.
@ -243,35 +195,61 @@ impl<M: AccessModel + Sized> Vfs<M> {
self.access_model.clear();
}
/// Resolve the real path for a file id.
pub fn file_path(&self, id: TypstFileId) -> Result<PathResolution, FileError> {
self.access_model.inner.resolver.path_for_id(id)
}
/// Reset the shadowing files in [`OverlayAccessModel`].
///
/// Note: This function is independent from [`Vfs::reset`].
pub fn reset_shadow(&mut self) {
self.access_model.clear_shadow();
self.access_model.inner.inner.clear_shadow();
}
/// Get paths to all the shadowing files in [`OverlayAccessModel`].
pub fn shadow_paths(&self) -> Vec<Arc<Path>> {
pub fn shadow_paths(&self) -> Vec<ImmutPath> {
self.access_model.inner.inner.file_paths()
}
/// Get paths to all the shadowing files in [`OverlayAccessModel`].
pub fn shadow_ids(&self) -> Vec<TypstFileId> {
self.access_model.file_paths()
}
/// Add a shadowing file to the [`OverlayAccessModel`].
pub fn map_shadow(&mut self, path: &Path, content: Bytes) -> FileResult<()> {
self.access_model.add_file(path.into(), content);
self.access_model
.inner
.inner
.add_file(path, content, |c| c.into());
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`].
pub fn remove_shadow(&mut self, path: &Path) {
self.access_model.remove_file(path);
self.access_model.inner.inner.remove_file(path);
}
/// Add a shadowing file to the [`OverlayAccessModel`] by file id.
pub fn map_shadow_by_id(&mut self, file_id: TypstFileId, content: Bytes) -> FileResult<()> {
self.access_model.add_file(&file_id, content, |c| *c);
Ok(())
}
/// Remove a shadowing file from the [`OverlayAccessModel`] by file id.
pub fn remove_shadow_by_id(&mut self, file_id: TypstFileId) {
self.access_model.remove_file(&file_id);
}
/// Let the vfs notify the access model with a filesystem event.
///
/// See [`NotifyAccessModel`] for more information.
pub fn notify_fs_event(&mut self, event: FilesystemEvent) {
self.access_model.inner.notify(event);
self.access_model.inner.inner.inner.notify(event);
}
/// Returns the overall memory usage for the stored files.
@ -279,36 +257,18 @@ impl<M: AccessModel + Sized> Vfs<M> {
0
}
/// Id of the given path if it exists in the `Vfs` and is not deleted.
pub fn file_id(&self, path: &Path) -> FileId {
self.paths.file_id(path)
}
/// Read a file.
pub fn read(&self, path: &Path) -> FileResult<Bytes> {
pub fn read(&self, path: TypstFileId) -> FileResult<Bytes> {
if self.access_model.is_file(path)? {
self.access_model.content(path)
} else {
Err(FileError::IsDirectory)
}
}
}
impl<M: AccessModel> FsProvider for Vfs<M> {
fn file_path(&self, id: FileId) -> ImmutPath {
self.paths.file_path(id)
}
fn mtime(&self, src: FileId) -> FileResult<Time> {
self.access_model.mtime(&self.file_path(src))
}
fn read(&self, src: FileId) -> FileResult<Bytes> {
self.access_model.content(&self.file_path(src))
}
fn is_file(&self, src: FileId) -> FileResult<bool> {
self.access_model.is_file(&self.file_path(src))
/// Whether the given path is a file.
pub fn is_file(&self, path: TypstFileId) -> FileResult<bool> {
self.access_model.is_file(path)
}
}

67
crates/tinymist-vfs/src/notify.rs
View file

@ -4,33 +4,20 @@ use std::path::Path;
use rpds::RedBlackTreeMapSync;
use typst::diag::{FileError, FileResult};
use crate::{AccessModel, Bytes, ImmutPath};
/// internal representation of [`NotifyFile`]
#[derive(Debug, Clone)]
struct NotifyFileRepr {
mtime: crate::Time,
content: Bytes,
}
use crate::{Bytes, ImmutPath, PathAccessModel};
/// A file snapshot that is notified by some external source
///
/// Note: The error is boxed to avoid large stack size
#[derive(Clone)]
pub struct FileSnapshot(Result<NotifyFileRepr, Box<FileError>>);
pub struct FileSnapshot(Result<Bytes, Box<FileError>>);
impl fmt::Debug for FileSnapshot {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.0.as_ref() {
Ok(v) => f
.debug_struct("FileSnapshot")
.field("mtime", &v.mtime)
.field(
"content",
&FileContent {
len: v.content.len(),
},
)
.field("content", &FileContent { len: v.len() })
.finish(),
Err(e) => f.debug_struct("FileSnapshot").field("error", &e).finish(),
}
@ -38,37 +25,25 @@ impl fmt::Debug for FileSnapshot {
}
impl FileSnapshot {
/// Access the internal data of the file snapshot
/// content of the file
#[inline]
#[track_caller]
fn retrieve<'a, T>(&'a self, f: impl FnOnce(&'a NotifyFileRepr) -> T) -> FileResult<T> {
self.0.as_ref().map(f).map_err(|e| *e.clone())
}
/// mtime of the file
pub fn mtime(&self) -> FileResult<&crate::Time> {
self.retrieve(|e| &e.mtime)
}
/// content of the file
pub fn content(&self) -> FileResult<&Bytes> {
self.retrieve(|e| &e.content)
self.0.as_ref().map_err(|e| *e.clone())
}
/// Whether the related file is a file
#[inline]
#[track_caller]
pub fn is_file(&self) -> FileResult<bool> {
self.retrieve(|_| true)
self.content().map(|_| true)
}
}
/// Convenient function to create a [`FileSnapshot`] from tuple
impl From<FileResult<(crate::Time, Bytes)>> for FileSnapshot {
fn from(result: FileResult<(crate::Time, Bytes)>) -> Self {
Self(
result
.map(|(mtime, content)| NotifyFileRepr { mtime, content })
.map_err(Box::new),
)
impl From<FileResult<Bytes>> for FileSnapshot {
fn from(result: FileResult<Bytes>) -> Self {
Self(result.map_err(Box::new))
}
}
@ -212,7 +187,7 @@ pub struct NotifyAccessModel<M> {
pub inner: M,
}
impl<M: AccessModel> NotifyAccessModel<M> {
impl<M: PathAccessModel> NotifyAccessModel<M> {
/// Create a new notify access model
pub fn new(inner: M) -> Self {
Self {
@ -238,15 +213,7 @@ impl<M: AccessModel> NotifyAccessModel<M> {
}
}
impl<M: AccessModel> AccessModel for NotifyAccessModel<M> {
fn mtime(&self, src: &Path) -> FileResult<crate::Time> {
if let Some(entry) = self.files.get(src) {
return entry.mtime().cloned();
}
self.inner.mtime(src)
}
impl<M: PathAccessModel> PathAccessModel for NotifyAccessModel<M> {
fn is_file(&self, src: &Path) -> FileResult<bool> {
if let Some(entry) = self.files.get(src) {
return entry.is_file();
@ -255,14 +222,6 @@ impl<M: AccessModel> AccessModel for NotifyAccessModel<M> {
self.inner.is_file(src)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
if self.files.contains_key(src) {
return Ok(src.into());
}
self.inner.real_path(src)
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
if let Some(entry) = self.files.get(src) {
return entry.content().cloned();

92
crates/tinymist-vfs/src/overlay.rs
View file

@ -1,28 +1,21 @@
use std::path::Path;
use std::sync::Arc;
use std::{borrow::Borrow, cmp::Ord, path::Path};
use rpds::RedBlackTreeMapSync;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{AccessModel, Bytes, Time};
#[derive(Debug, Clone)]
struct OverlayFileMeta {
mt: Time,
content: Bytes,
}
use crate::{AccessModel, Bytes, PathAccessModel, TypstFileId};
/// Provides overlay access model which allows to shadow the underlying access
/// model with memory contents.
#[derive(Default, Debug, Clone)]
pub struct OverlayAccessModel<M> {
files: RedBlackTreeMapSync<Arc<Path>, OverlayFileMeta>,
pub struct OverlayAccessModel<K: Ord, M> {
files: RedBlackTreeMapSync<K, Bytes>,
/// The underlying access model
pub inner: M,
}
impl<M: AccessModel> OverlayAccessModel<M> {
impl<K: Ord + Clone, M> OverlayAccessModel<K, M> {
/// Create a new [`OverlayAccessModel`] with the given inner access model
pub fn new(inner: M) -> Self {
Self {
@ -47,54 +40,35 @@ impl<M: AccessModel> OverlayAccessModel<M> {
}
/// Get the shadowed file paths
pub fn file_paths(&self) -> Vec<Arc<Path>> {
pub fn file_paths(&self) -> Vec<K> {
self.files.keys().cloned().collect()
}
/// Add a shadow file to the [`OverlayAccessModel`]
pub fn add_file(&mut self, path: Arc<Path>, content: Bytes) {
// we change mt every time, since content almost changes every time
// Note: we can still benefit from cache, since we incrementally parse source
let mt = tinymist_std::time::now();
let meta = OverlayFileMeta { mt, content };
match self.files.get_mut(&path) {
pub fn add_file<Q: Ord + ?Sized>(&mut self, path: &Q, content: Bytes, cast: impl Fn(&Q) -> K)
where
K: Borrow<Q>,
{
match self.files.get_mut(path) {
Some(e) => {
if e.mt == meta.mt && e.content != meta.content {
e.mt = meta
.mt
// [`crate::Time`] has a minimum resolution of 1ms
// we negate the time by 1ms so that the time is always
// invalidated
.checked_sub(std::time::Duration::from_millis(1))
.unwrap();
e.content = meta.content.clone();
} else {
*e = meta.clone();
}
*e = content;
}
None => {
self.files.insert_mut(path, meta);
self.files.insert_mut(cast(path), content);
}
}
}
/// Remove a shadow file from the [`OverlayAccessModel`]
pub fn remove_file(&mut self, path: &Path) {
pub fn remove_file<Q: Ord + ?Sized>(&mut self, path: &Q)
where
K: Borrow<Q>,
{
self.files.remove_mut(path);
}
}
impl<M: AccessModel> AccessModel for OverlayAccessModel<M> {
fn mtime(&self, src: &Path) -> FileResult<Time> {
if let Some(meta) = self.files.get(src) {
return Ok(meta.mt);
}
self.inner.mtime(src)
}
impl<M: PathAccessModel> PathAccessModel for OverlayAccessModel<ImmutPath, M> {
fn is_file(&self, src: &Path) -> FileResult<bool> {
if self.files.get(src).is_some() {
return Ok(true);
@ -103,17 +77,27 @@ impl<M: AccessModel> AccessModel for OverlayAccessModel<M> {
self.inner.is_file(src)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
if self.files.get(src).is_some() {
return Ok(src.into());
}
self.inner.real_path(src)
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
if let Some(meta) = self.files.get(src) {
return Ok(meta.content.clone());
if let Some(content) = self.files.get(src) {
return Ok(content.clone());
}
self.inner.content(src)
}
}
impl<M: AccessModel> AccessModel for OverlayAccessModel<TypstFileId, M> {
fn is_file(&self, src: TypstFileId) -> FileResult<bool> {
if self.files.get(&src).is_some() {
return Ok(true);
}
self.inner.is_file(src)
}
fn content(&self, src: TypstFileId) -> FileResult<Bytes> {
if let Some(content) = self.files.get(&src) {
return Ok(content.clone());
}
self.inner.content(src)

71
crates/tinymist-vfs/src/path_interner.rs
View file

@ -1,71 +0,0 @@
//! Maps paths to compact integer ids. We don't care about clearings paths which
//! no longer exist -- the assumption is total size of paths we ever look at is
//! not too big.
use std::hash::{BuildHasherDefault, Hash};
use indexmap::IndexMap;
use tinymist_std::hash::FxHasher;
use super::FileId;
/// Structure to map between [`VfsPath`] and [`FileId`].
#[derive(Debug)]
pub(crate) struct PathInterner<P, Ext = ()> {
map: IndexMap<P, Ext, BuildHasherDefault<FxHasher>>,
}
impl<P, Ext> Default for PathInterner<P, Ext> {
fn default() -> Self {
Self {
map: IndexMap::default(),
}
}
}
impl<P: Hash + Eq, Ext> PathInterner<P, Ext> {
/// Scan through each value in the set and keep those where the
/// closure `keep` returns `true`.
///
/// The elements are visited in order, and remaining elements keep their
/// order.
///
/// Computes in **O(n)** time (average).
#[allow(dead_code)]
pub fn retain(&mut self, keep: impl FnMut(&P, &mut Ext) -> bool) {
self.map.retain(keep)
}
/// Insert `path` in `self`.
///
/// - If `path` already exists in `self`, returns its associated id;
/// - Else, returns a newly allocated id.
#[inline]
pub(crate) fn intern(&mut self, path: P, ext: Ext) -> (FileId, Option<&mut Ext>) {
let (id, _) = self.map.insert_full(path, ext);
assert!(id < u32::MAX as usize);
(FileId(id as u32), None)
}
/// Returns the path corresponding to `id`.
///
/// # Panics
///
/// Panics if `id` does not exists in `self`.
#[allow(dead_code)]
pub(crate) fn lookup(&self, id: FileId) -> &P {
self.map.get_index(id.0 as usize).unwrap().0
}
}
#[cfg(test)]
mod tests {
use super::PathInterner;
use std::path::PathBuf;
#[test]
fn test_interner_path_buf() {
let mut interner = PathInterner::<PathBuf>::default();
let (id, ..) = interner.intern(PathBuf::from("foo"), ());
assert_eq!(interner.lookup(id), &PathBuf::from("foo"));
}
}

286
crates/tinymist-vfs/src/path_mapper.rs Normal file
View file

@ -0,0 +1,286 @@
//! Maps paths to compact integer ids. We don't care about clearings paths which
//! no longer exist -- the assumption is total size of paths we ever look at is
//! not too big.
use core::fmt;
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use parking_lot::RwLock;
use tinymist_std::path::PathClean;
use tinymist_std::ImmutPath;
use typst::diag::{eco_format, EcoString, FileError, FileResult};
use typst::syntax::package::{PackageSpec, PackageVersion};
use typst::syntax::VirtualPath;
use super::TypstFileId;
pub enum PathResolution {
Resolved(PathBuf),
Rootless(Cow<'static, VirtualPath>),
}
impl PathResolution {
pub fn to_err(self) -> FileResult<PathBuf> {
match self {
PathResolution::Resolved(path) => Ok(path),
PathResolution::Rootless(_) => Err(FileError::AccessDenied),
}
}
pub fn as_path(&self) -> &Path {
match self {
PathResolution::Resolved(path) => path.as_path(),
PathResolution::Rootless(path) => path.as_rooted_path(),
}
}
pub fn join(&self, path: &str) -> FileResult<PathResolution> {
match self {
PathResolution::Resolved(path) => Ok(PathResolution::Resolved(path.join(path))),
PathResolution::Rootless(root) => {
Ok(PathResolution::Rootless(Cow::Owned(root.join(path))))
}
}
}
}
pub trait RootResolver {
fn path_for_id(&self, file_id: TypstFileId) -> FileResult<PathResolution> {
use WorkspaceResolution::*;
let root = match WorkspaceResolver::resolve(file_id)? {
Workspace(id) => id.path().clone(),
Package => {
self.resolve_package_root(file_id.package().expect("not a file in package"))?
}
UntitledRooted(..) | Rootless => {
return Ok(PathResolution::Rootless(Cow::Borrowed(file_id.vpath())))
}
};
file_id
.vpath()
.resolve(&root)
.map(PathResolution::Resolved)
.ok_or_else(|| FileError::AccessDenied)
}
fn resolve_root(&self, file_id: TypstFileId) -> FileResult<Option<ImmutPath>> {
use WorkspaceResolution::*;
match WorkspaceResolver::resolve(file_id)? {
Workspace(id) | UntitledRooted(id) => Ok(Some(id.path().clone())),
Rootless => Ok(None),
Package => self
.resolve_package_root(file_id.package().expect("not a file in package"))
.map(Some),
}
}
fn resolve_package_root(&self, pkg: &PackageSpec) -> FileResult<ImmutPath>;
}
#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct WorkspaceId(u16);
const NO_VERSION: PackageVersion = PackageVersion {
major: 0,
minor: 0,
patch: 0,
};
const UNTITLED_ROOT: PackageVersion = PackageVersion {
major: 0,
minor: 0,
patch: 1,
};
impl WorkspaceId {
fn package(&self) -> PackageSpec {
PackageSpec {
namespace: WorkspaceResolver::WORKSPACE_NS.clone(),
name: eco_format!("p{}", self.0),
version: NO_VERSION,
}
}
fn untitled_root(&self) -> PackageSpec {
PackageSpec {
namespace: WorkspaceResolver::WORKSPACE_NS.clone(),
name: eco_format!("p{}", self.0),
version: UNTITLED_ROOT,
}
}
pub fn path(&self) -> ImmutPath {
let interner = INTERNER.read();
interner
.from_id
.get(self.0 as usize)
.expect("invalid workspace id")
.clone()
}
fn from_package_name(name: &str) -> Option<WorkspaceId> {
if !name.starts_with("p") {
return None;
}
let num = name[1..].parse().ok()?;
Some(WorkspaceId(num))
}
}
/// The global package-path interner.
static INTERNER: LazyLock<RwLock<Interner>> = LazyLock::new(|| {
RwLock::new(Interner {
to_id: HashMap::new(),
from_id: Vec::new(),
})
});
pub enum WorkspaceResolution {
Workspace(WorkspaceId),
UntitledRooted(WorkspaceId),
Rootless,
Package,
}
/// A package-path interner.
struct Interner {
to_id: HashMap<ImmutPath, WorkspaceId>,
from_id: Vec<ImmutPath>,
}
#[derive(Default)]
pub struct WorkspaceResolver {}
impl WorkspaceResolver {
pub const WORKSPACE_NS: EcoString = EcoString::inline("ws");
pub fn is_workspace_file(fid: TypstFileId) -> bool {
fid.package()
.is_some_and(|p| p.namespace == WorkspaceResolver::WORKSPACE_NS)
}
pub fn is_package_file(fid: TypstFileId) -> bool {
fid.package()
.is_some_and(|p| p.namespace != WorkspaceResolver::WORKSPACE_NS)
}
/// Id of the given path if it exists in the `Vfs` and is not deleted.
pub fn workspace_id(root: &ImmutPath) -> WorkspaceId {
// Try to find an existing entry that we can reuse.
//
// We could check with just a read lock, but if the pair is not yet
// present, we would then need to recheck after acquiring a write lock,
// which is probably not worth it.
let mut interner = INTERNER.write();
if let Some(&id) = interner.to_id.get(root) {
return id;
}
let root = ImmutPath::from(root.clean());
// Create a new entry forever by leaking the pair. We can't leak more
// than 2^16 pair (and typically will leak a lot less), so its not a
// big deal.
let num = interner.from_id.len().try_into().expect("out of file ids");
let id = WorkspaceId(num);
interner.to_id.insert(root.clone(), id);
interner.from_id.push(root.clone());
id
}
/// Creates a file id for a rootless file.
pub fn rootless_file(path: VirtualPath) -> TypstFileId {
TypstFileId::new(None, path)
}
/// Creates a file id for a rootless file.
pub fn file_with_parent_root(path: &Path) -> Option<TypstFileId> {
if !path.is_absolute() {
return None;
}
let parent = path.parent()?;
let parent = ImmutPath::from(parent);
let path = VirtualPath::new(path.file_name()?);
Some(Self::workspace_file(Some(&parent), path))
}
/// Creates a file id for a file in some workspace. The `root` is the root
/// directory of the workspace. If `root` is `None`, the source code at the
/// `path` will not be able to access physical files.
pub fn workspace_file(root: Option<&ImmutPath>, path: VirtualPath) -> TypstFileId {
let workspace = root.map(Self::workspace_id);
TypstFileId::new(workspace.as_ref().map(WorkspaceId::package), path)
}
/// Mounts an untiled file to some workspace. The `root` is the
/// root directory of the workspace. If `root` is `None`, the source
/// code at the `path` will not be able to access physical files.
pub fn rooted_untitled(root: Option<&ImmutPath>, path: VirtualPath) -> TypstFileId {
let workspace = root.map(Self::workspace_id);
TypstFileId::new(workspace.as_ref().map(WorkspaceId::untitled_root), path)
}
/// File path corresponding to the given `fid`.
pub fn resolve(fid: TypstFileId) -> FileResult<WorkspaceResolution> {
let Some(package) = fid.package() else {
return Ok(WorkspaceResolution::Rootless);
};
match package.namespace.as_str() {
"ws" => {
let id = WorkspaceId::from_package_name(&package.name).ok_or_else(|| {
FileError::Other(Some(eco_format!("bad workspace id: {fid:?}")))
})?;
Ok(if package.version == UNTITLED_ROOT {
WorkspaceResolution::UntitledRooted(id)
} else {
WorkspaceResolution::Workspace(id)
})
}
_ => Ok(WorkspaceResolution::Package),
}
}
/// File path corresponding to the given `fid`.
pub fn display(id: Option<TypstFileId>) -> Resolving {
Resolving { id }
}
}
pub struct Resolving {
id: Option<TypstFileId>,
}
impl fmt::Debug for Resolving {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
use WorkspaceResolution::*;
let Some(id) = self.id else {
return write!(f, "None");
};
let path = match WorkspaceResolver::resolve(id) {
Ok(Workspace(workspace)) => id.vpath().resolve(&workspace.path()),
Ok(UntitledRooted(..)) => Some(id.vpath().as_rootless_path().to_owned()),
Ok(Rootless | Package) | Err(_) => None,
};
if let Some(path) = path {
write!(f, "{}", path.display())
} else {
write!(f, "{:?}", self.id)
}
}
}
#[cfg(test)]
mod tests {
#[test]
fn test_interner_untitled() {}
}

30
crates/tinymist-vfs/src/resolve.rs Normal file
View file

@ -0,0 +1,30 @@
use std::{fmt::Debug, sync::Arc};
use typst::diag::FileResult;
use crate::{path_mapper::RootResolver, AccessModel, Bytes, PathAccessModel, TypstFileId};
/// Provides resolve access model.
#[derive(Clone)]
pub struct ResolveAccessModel<M> {
pub resolver: Arc<dyn RootResolver + Send + Sync>,
pub inner: M,
}
impl<M> Debug for ResolveAccessModel<M> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ResolveAccessModel").finish()
}
}
impl<M: PathAccessModel> AccessModel for ResolveAccessModel<M> {
fn is_file(&self, fid: TypstFileId) -> FileResult<bool> {
self.inner
.is_file(&self.resolver.path_for_id(fid)?.to_err()?)
}
fn content(&self, fid: TypstFileId) -> FileResult<Bytes> {
self.inner
.content(&self.resolver.path_for_id(fid)?.to_err()?)
}
}

17
crates/tinymist-vfs/src/system.rs
View file

@ -2,8 +2,8 @@ use std::{fs::File, io::Read, path::Path};
use typst::diag::{FileError, FileResult};
use crate::{AccessModel, Bytes, Time};
use tinymist_std::{ImmutPath, ReadAllOnce};
use crate::{Bytes, PathAccessModel};
use tinymist_std::ReadAllOnce;
/// Provides SystemAccessModel that makes access to the local file system for
/// system compilation.
@ -14,27 +14,17 @@ impl SystemAccessModel {
fn stat(&self, src: &Path) -> std::io::Result<SystemFileMeta> {
let meta = std::fs::metadata(src)?;
Ok(SystemFileMeta {
mt: meta.modified()?,
is_file: meta.is_file(),
})
}
}
impl AccessModel for SystemAccessModel {
fn mtime(&self, src: &Path) -> FileResult<Time> {
let f = |e| FileError::from_io(e, src);
Ok(self.stat(src).map_err(f)?.mt)
}
impl PathAccessModel for SystemAccessModel {
fn is_file(&self, src: &Path) -> FileResult<bool> {
let f = |e| FileError::from_io(e, src);
Ok(self.stat(src).map_err(f)?.is_file)
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
Ok(src.into())
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
let f = |e| FileError::from_io(e, src);
let mut buf = Vec::<u8>::new();
@ -78,6 +68,5 @@ impl ReadAllOnce for LazyFile {
/// Meta data of a file in the local file system.
#[derive(Debug, Clone, Copy)]
pub struct SystemFileMeta {
mt: std::time::SystemTime,
is_file: bool,
}

34
crates/tinymist-vfs/src/trace.rs
View file

@ -1,9 +1,8 @@
use std::{path::Path, sync::atomic::AtomicU64};
use std::sync::atomic::AtomicU64;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
use crate::{AccessModel, Bytes};
use crate::{AccessModel, Bytes, TypstFileId};
/// Provides trace access model which traces the underlying access model.
///
@ -30,20 +29,7 @@ impl<M: AccessModel + Sized> AccessModel for TraceAccessModel<M> {
self.inner.clear();
}
fn mtime(&self, src: &Path) -> FileResult<crate::Time> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.mtime(src);
let elapsed = instant.elapsed();
// self.trace[0] += elapsed.as_nanos() as u64;
self.trace[0].fetch_add(
elapsed.as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
crate::utils::console_log!("mtime: {:?} {:?} => {:?}", src, elapsed, res);
res
}
fn is_file(&self, src: &Path) -> FileResult<bool> {
fn is_file(&self, src: TypstFileId) -> FileResult<bool> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.is_file(src);
let elapsed = instant.elapsed();
@ -55,19 +41,7 @@ impl<M: AccessModel + Sized> AccessModel for TraceAccessModel<M> {
res
}
fn real_path(&self, src: &Path) -> FileResult<ImmutPath> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.real_path(src);
let elapsed = instant.elapsed();
self.trace[2].fetch_add(
elapsed.as_nanos() as u64,
std::sync::atomic::Ordering::Relaxed,
);
crate::utils::console_log!("real_path: {:?} {:?}", src, elapsed);
res
}
fn content(&self, src: &Path) -> FileResult<Bytes> {
fn content(&self, src: TypstFileId) -> FileResult<Bytes> {
let instant = tinymist_std::time::Instant::now();
let res = self.inner.content(src);
let elapsed = instant.elapsed();

6
crates/tinymist-world/src/browser.rs
View file

@ -7,6 +7,7 @@ use typst::utils::LazyHash;
use crate::entry::EntryState;
use crate::font::FontResolverImpl;
use crate::package::browser::ProxyRegistry;
use crate::package::RegistryPathMapper;
/// A world that provides access to the browser.
/// It is under development.
@ -39,7 +40,10 @@ impl TypstBrowserUniverse {
registry: ProxyRegistry,
font_resolver: FontResolverImpl,
) -> Self {
let vfs = tinymist_vfs::Vfs::new(access_model);
let registry = Arc::new(registry);
let resolver = Arc::new(RegistryPathMapper::new(registry.clone()));
let vfs = tinymist_vfs::Vfs::new(resolver, access_model);
Self::new_raw(
EntryState::new_rooted(root_dir.into(), None),

102
crates/tinymist-world/src/entry.rs
View file

@ -1,18 +1,15 @@
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock};
use std::sync::LazyLock;
use serde::{Deserialize, Serialize};
use tinymist_std::{error::prelude::*, ImmutPath};
use tinymist_vfs::{WorkspaceResolution, WorkspaceResolver};
use typst::diag::SourceResult;
use typst::syntax::{FileId, VirtualPath};
pub trait EntryReader {
fn entry_state(&self) -> EntryState;
fn workspace_root(&self) -> Option<Arc<Path>> {
self.entry_state().root().clone()
}
fn main_id(&self) -> Option<FileId> {
self.entry_state().main()
}
@ -28,10 +25,6 @@ pub trait EntryManager: EntryReader {
#[derive(Debug, Clone, Hash, PartialEq, Eq, Default)]
pub struct EntryState {
/// The differences is that: if the entry is rooted, the workspace root is
/// the parent of the entry file and cannot be used by workspace functions
/// like [`EntryState::try_select_path_in_workspace`].
rooted: bool,
/// Path to the root directory of compilation.
/// The world forbids direct access to files outside this directory.
root: Option<ImmutPath>,
@ -49,7 +42,6 @@ impl EntryState {
/// Create an entry state with no workspace root and no main file.
pub fn new_detached() -> Self {
Self {
rooted: false,
root: None,
main: None,
}
@ -60,21 +52,37 @@ impl EntryState {
Self::new_rooted(root, None)
}
/// Create an entry state with a workspace root and an optional main file.
pub fn new_rooted(root: ImmutPath, main: Option<FileId>) -> Self {
/// Create an entry state without permission to access the file system.
pub fn new_rootless(main: VirtualPath) -> Self {
Self {
root: None,
main: Some(FileId::new(None, main)),
}
}
/// Create an entry state with a workspace root and an main file.
pub fn new_rooted_by_id(root: ImmutPath, main: FileId) -> Self {
Self::new_rooted(root, Some(main.vpath().clone()))
}
/// Create an entry state with a workspace root and an optional main file.
pub fn new_rooted(root: ImmutPath, main: Option<VirtualPath>) -> Self {
let main = main.map(|main| WorkspaceResolver::workspace_file(Some(&root), main));
Self {
rooted: true,
root: Some(root),
main,
}
}
/// Create an entry state with only a main file given.
pub fn new_rootless(entry: ImmutPath) -> Option<Self> {
pub fn new_rooted_by_parent(entry: ImmutPath) -> Option<Self> {
let root = entry.parent().map(ImmutPath::from);
let main =
WorkspaceResolver::workspace_file(root.as_ref(), VirtualPath::new(entry.file_name()?));
Some(Self {
rooted: false,
root: entry.parent().map(From::from),
main: Some(FileId::new(None, VirtualPath::new(entry.file_name()?))),
root,
main: Some(main),
})
}
@ -87,27 +95,34 @@ impl EntryState {
}
pub fn workspace_root(&self) -> Option<ImmutPath> {
self.rooted.then(|| self.root.clone()).flatten()
if let Some(main) = self.main {
match WorkspaceResolver::resolve(main).ok()? {
WorkspaceResolution::Workspace(id) | WorkspaceResolution::UntitledRooted(id) => {
Some(id.path().clone())
}
WorkspaceResolution::Rootless => None,
WorkspaceResolution::Package => self.root.clone(),
}
} else {
self.root.clone()
}
}
pub fn select_in_workspace(&self, id: FileId) -> EntryState {
pub fn select_in_workspace(&self, id: &Path) -> EntryState {
let id = WorkspaceResolver::workspace_file(self.root.as_ref(), VirtualPath::new(id));
Self {
rooted: self.rooted,
root: self.root.clone(),
main: Some(id),
}
}
pub fn try_select_path_in_workspace(
&self,
p: &Path,
allow_rootless: bool,
) -> ZResult<Option<EntryState>> {
pub fn try_select_path_in_workspace(&self, p: &Path) -> ZResult<Option<EntryState>> {
Ok(match self.workspace_root() {
Some(root) => match p.strip_prefix(&root) {
Ok(p) => Some(EntryState::new_rooted(
root.clone(),
Some(FileId::new(None, VirtualPath::new(p))),
Some(VirtualPath::new(p)),
)),
Err(e) => {
return Err(
@ -115,8 +130,7 @@ impl EntryState {
)
}
},
None if allow_rootless => EntryState::new_rootless(p.into()),
None => None,
None => EntryState::new_rooted_by_parent(p.into()),
})
}
@ -138,11 +152,9 @@ pub enum EntryOpts {
/// Relative path to the main file in the workspace.
entry: Option<PathBuf>,
},
RootlessEntry {
RootByParent {
/// Path to the entry file of compilation.
entry: PathBuf,
/// Parent directory of the entry file.
root: Option<PathBuf>,
},
Detached,
}
@ -171,9 +183,8 @@ impl EntryOpts {
return None;
}
Some(Self::RootlessEntry {
Some(Self::RootByParent {
entry: entry.clone(),
root: entry.parent().map(From::from),
})
}
}
@ -185,33 +196,16 @@ impl TryFrom<EntryOpts> for EntryState {
match value {
EntryOpts::Workspace { root, entry } => Ok(EntryState::new_rooted(
root.as_path().into(),
entry.map(|e| FileId::new(None, VirtualPath::new(e))),
entry.map(VirtualPath::new),
)),
EntryOpts::RootlessEntry { entry, root } => {
EntryOpts::RootByParent { entry } => {
if entry.is_relative() {
return Err(error_once!("entry path must be absolute", path: entry.display()));
}
// todo: is there path that has no parent?
let root = root
.as_deref()
.or_else(|| entry.parent())
.ok_or_else(|| error_once!("a root must be determined for EntryOpts::PreparedEntry", path: entry.display()))?;
let relative_entry = match entry.strip_prefix(root) {
Ok(e) => e,
Err(_) => {
return Err(
error_once!("entry path must be inside the root", path: entry.display()),
)
}
};
Ok(EntryState {
rooted: false,
root: Some(root.into()),
main: Some(FileId::new(None, VirtualPath::new(relative_entry))),
})
EntryState::new_rooted_by_parent(entry.as_path().into())
.ok_or_else(|| error_once!("entry path is invalid", path: entry.display()))
}
EntryOpts::Detached => Ok(EntryState::new_detached()),
}

50
crates/tinymist-world/src/lib.rs
View file

@ -32,32 +32,23 @@ pub(crate) mod browser;
#[cfg(feature = "browser")]
pub use browser::{BrowserCompilerFeat, TypstBrowserUniverse, TypstBrowserWorld};
use std::{
path::{Path, PathBuf},
sync::Arc,
};
use std::{path::Path, sync::Arc};
use ecow::EcoVec;
use tinymist_std::ImmutPath;
use tinymist_vfs::AccessModel as VfsAccessModel;
use typst::{
diag::{At, FileResult, SourceResult},
foundations::Bytes,
syntax::FileId,
syntax::Span,
};
use tinymist_vfs::PathAccessModel as VfsAccessModel;
use typst::diag::{At, FileResult, SourceResult};
use typst::foundations::Bytes;
use typst::syntax::{FileId, Span};
use font::FontResolver;
use package::PackageRegistry;
/// Latest version of the shadow api, which is in beta.
pub trait ShadowApi {
fn _shadow_map_id(&self, _file_id: FileId) -> FileResult<PathBuf> {
unimplemented!()
}
/// Get the shadow files.
fn shadow_paths(&self) -> Vec<Arc<Path>>;
/// Get the shadow files by file id.
fn shadow_ids(&self) -> Vec<FileId>;
/// Reset the shadow files.
fn reset_shadow(&mut self) {
@ -73,20 +64,14 @@ pub trait ShadowApi {
fn unmap_shadow(&mut self, path: &Path) -> FileResult<()>;
/// Add a shadow file to the driver by file id.
/// Note: to enable this function, `ShadowApi` must implement
/// `_shadow_map_id`.
fn map_shadow_by_id(&mut self, file_id: FileId, content: Bytes) -> FileResult<()> {
let file_path = self._shadow_map_id(file_id)?;
self.map_shadow(&file_path, content)
}
/// Note: If a *path* is both shadowed by id and by path, the shadow by id
/// will be used.
fn map_shadow_by_id(&mut self, file_id: FileId, content: Bytes) -> FileResult<()>;
/// Add a shadow file to the driver by file id.
/// Note: to enable this function, `ShadowApi` must implement
/// `_shadow_map_id`.
fn unmap_shadow_by_id(&mut self, file_id: FileId) -> FileResult<()> {
let file_path = self._shadow_map_id(file_id)?;
self.unmap_shadow(&file_path)
}
/// Note: If a *path* is both shadowed by id and by path, the shadow by id
/// will be used.
fn unmap_shadow_by_id(&mut self, file_id: FileId) -> FileResult<()>;
}
pub trait ShadowApiExt {
@ -134,14 +119,17 @@ impl<C: ShadowApi> ShadowApiExt for C {
content: Bytes,
f: impl FnOnce(&mut Self) -> SourceResult<T>,
) -> SourceResult<T> {
let file_path = self._shadow_map_id(file_id).at(Span::detached())?;
self.with_shadow_file(&file_path, content, f)
self.map_shadow_by_id(file_id, content)
.at(Span::detached())?;
let res: Result<T, EcoVec<typst::diag::SourceDiagnostic>> = f(self);
self.unmap_shadow_by_id(file_id).at(Span::detached())?;
res
}
}
/// Latest version of the world dependencies api, which is in beta.
pub trait WorldDeps {
fn iter_dependencies(&self, f: &mut dyn FnMut(ImmutPath));
fn iter_dependencies(&self, f: &mut dyn FnMut(FileId));
}
/// type trait interface of [`CompilerWorld`].

18
crates/tinymist-world/src/package/mod.rs
View file

@ -2,6 +2,8 @@ impl Notifier for DummyNotifier {}
use std::{path::Path, sync::Arc};
use ecow::EcoString;
use tinymist_std::ImmutPath;
use typst::diag::FileResult;
pub use typst::diag::PackageError;
pub use typst::syntax::package::PackageSpec;
@ -29,6 +31,22 @@ pub trait PackageRegistry {
}
}
pub struct RegistryPathMapper<T> {
pub registry: Arc<T>,
}
impl<T> RegistryPathMapper<T> {
pub fn new(registry: Arc<T>) -> Self {
Self { registry }
}
}
impl<T: PackageRegistry> tinymist_vfs::RootResolver for RegistryPathMapper<T> {
fn resolve_package_root(&self, pkg: &PackageSpec) -> FileResult<ImmutPath> {
Ok(self.registry.resolve(pkg)?)
}
}
pub trait Notifier {
fn downloading(&self, _spec: &PackageSpec) {}
}

30
crates/tinymist-world/src/source.rs
View file

@ -5,8 +5,8 @@ use std::{num::NonZeroUsize, sync::Arc};
use parking_lot::{Mutex, RwLock};
use tinymist_std::hash::FxHashMap;
use tinymist_std::{ImmutPath, QueryRef};
use tinymist_vfs::{Bytes, FileId, FsProvider, TypstFileId};
use tinymist_std::QueryRef;
use tinymist_vfs::{Bytes, FsProvider, TypstFileId};
use typst::{
diag::{FileError, FileResult},
syntax::Source,
@ -56,7 +56,7 @@ impl<T: Revised> SharedState<T> {
pub struct SourceCache {
last_accessed_rev: NonZeroUsize,
fid: FileId,
fid: TypstFileId,
source: IncrFileQuery<Source>,
buffer: FileQuery<Bytes>,
}
@ -151,27 +151,23 @@ impl SourceDb {
///
/// When you don't reset the vfs for each compilation, this function will
/// still return remaining files from the previous compilation.
pub fn iter_dependencies_dyn<'a>(
&'a self,
p: &'a impl FsProvider,
f: &mut dyn FnMut(ImmutPath),
) {
pub fn iter_dependencies_dyn(&self, f: &mut dyn FnMut(TypstFileId)) {
for slot in self.slots.lock().iter() {
f(p.file_path(slot.1.fid));
f(slot.1.fid);
}
}
/// Get file content by path.
pub fn file(&self, id: TypstFileId, fid: FileId, p: &impl FsProvider) -> FileResult<Bytes> {
self.slot(id, fid, |slot| slot.buffer.compute(|| p.read(fid)).cloned())
pub fn file(&self, fid: TypstFileId, p: &impl FsProvider) -> FileResult<Bytes> {
self.slot(fid, |slot| slot.buffer.compute(|| p.read(fid)).cloned())
}
/// Get source content by path and assign the source with a given typst
/// global file id.
///
/// See `Vfs::resolve_with_f` for more information.
pub fn source(&self, id: TypstFileId, fid: FileId, p: &impl FsProvider) -> FileResult<Source> {
self.slot(id, fid, |slot| {
pub fn source(&self, fid: TypstFileId, p: &impl FsProvider) -> FileResult<Source> {
self.slot(fid, |slot| {
slot.source
.compute_with_context(|prev| {
let content = p.read(fid)?;
@ -184,7 +180,7 @@ impl SourceDb {
Ok(source)
}
// Return a new source if we don't have a reparse feature or no prev
_ => Ok(Source::new(id, next)),
_ => Ok(Source::new(fid, next)),
}
})
.cloned()
@ -192,11 +188,11 @@ impl SourceDb {
}
/// Insert a new slot into the vfs.
fn slot<T>(&self, id: TypstFileId, fid: FileId, f: impl FnOnce(&SourceCache) -> T) -> T {
fn slot<T>(&self, fid: TypstFileId, f: impl FnOnce(&SourceCache) -> T) -> T {
let mut slots = self.slots.lock();
f(slots.entry(id).or_insert_with(|| {
f(slots.entry(fid).or_insert_with(|| {
let state = self.shared.read();
let cache_entry = state.cache_entries.get(&id);
let cache_entry = state.cache_entries.get(&fid);
cache_entry
.map(|e| SourceCache {

8
crates/tinymist-world/src/system.rs
View file

@ -7,7 +7,7 @@ use typst::utils::LazyHash;
use crate::{
config::CompileOpts,
font::{system::SystemFontSearcher, FontResolverImpl},
package::http::HttpRegistry,
package::{http::HttpRegistry, RegistryPathMapper},
};
/// type trait of [`TypstSystemWorld`].
@ -33,12 +33,14 @@ impl TypstSystemUniverse {
/// See SystemCompilerFeat for instantiation details.
/// See [`CompileOpts`] for available options.
pub fn new(mut opts: CompileOpts) -> ZResult<Self> {
let registry: Arc<HttpRegistry> = Arc::default();
let resolver = Arc::new(RegistryPathMapper::new(registry.clone()));
let inputs = std::mem::take(&mut opts.inputs);
Ok(Self::new_raw(
opts.entry.clone().try_into()?,
Some(Arc::new(LazyHash::new(inputs))),
Vfs::new(SystemAccessModel {}),
HttpRegistry::default(),
Vfs::new(resolver, SystemAccessModel {}),
registry,
Arc::new(Self::resolve_fonts(opts)?),
))
}

154
crates/tinymist-world/src/world.rs
View file

@ -8,12 +8,12 @@ use std::{
use chrono::{DateTime, Datelike, Local};
use parking_lot::RwLock;
use tinymist_std::error::prelude::*;
use tinymist_std::ImmutPath;
use tinymist_vfs::{notify::FilesystemEvent, Vfs};
use tinymist_vfs::{notify::FilesystemEvent, PathResolution, Vfs, WorkspaceResolver};
use tinymist_vfs::{FsProvider, TypstFileId};
use typst::{
diag::{eco_format, At, EcoString, FileError, FileResult, SourceResult},
foundations::{Bytes, Datetime, Dict},
syntax::{FileId, Source, Span},
syntax::{FileId, Source, Span, VirtualPath},
text::{Font, FontBook},
utils::LazyHash,
Library, World,
@ -129,7 +129,7 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
entry: EntryState,
inputs: Option<Arc<LazyHash<Dict>>>,
vfs: Vfs<F::AccessModel>,
registry: F::Registry,
registry: Arc<F::Registry>,
font_resolver: Arc<F::FontResolver>,
) -> Self {
Self {
@ -140,7 +140,7 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
shared: Arc::new(RwLock::new(SharedState::default())),
font_resolver,
registry: Arc::new(registry),
registry,
vfs,
}
}
@ -202,7 +202,7 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
fn set_entry_file_(&mut self, entry_file: Arc<Path>) -> SourceResult<()> {
let state = self.entry_state();
let state = state
.try_select_path_in_workspace(&entry_file, true)
.try_select_path_in_workspace(&entry_file)
.map_err(|e| eco_format!("cannot select entry file out of workspace: {e}"))
.at(Span::detached())?
.ok_or_else(|| eco_format!("failed to determine root"))
@ -221,24 +221,17 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
}
/// Resolve the real path for a file id.
pub fn path_for_id(&self, id: FileId) -> Result<PathBuf, FileError> {
if id == *DETACHED_ENTRY {
return Ok(DETACHED_ENTRY.vpath().as_rooted_path().to_owned());
}
pub fn path_for_id(&self, id: FileId) -> Result<PathResolution, FileError> {
self.vfs.file_path(id)
}
// Determine the root path relative to which the file path
// will be resolved.
let root = match id.package() {
Some(spec) => self.registry.resolve(spec)?,
None => self.entry.root().ok_or(FileError::Other(Some(eco_format!(
"cannot access directory without root: state: {:?}",
self.entry
))))?,
};
// Join the path to the root. If it tries to escape, deny
// access. Note: It can still escape via symlinks.
id.vpath().resolve(&root).ok_or(FileError::AccessDenied)
/// Resolve the root of the workspace.
pub fn id_for_path(&self, path: &Path) -> Option<FileId> {
let root = self.entry.workspace_root()?;
Some(WorkspaceResolver::workspace_file(
Some(&root),
VirtualPath::new(path.strip_prefix(&root).ok()?),
))
}
pub fn get_semantic_token_legend(&self) -> Arc<SemanticTokensLegend> {
@ -255,7 +248,7 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
let path = Path::new(&e);
let s = self
.entry_state()
.try_select_path_in_workspace(path, true)?
.try_select_path_in_workspace(path)?
.ok_or_else(|| error_once!("cannot select file", path: e))?;
self.snapshot_with(Some(TaskInputs {
@ -275,18 +268,16 @@ impl<F: CompilerFeat> CompilerUniverse<F> {
impl<F: CompilerFeat> ShadowApi for CompilerUniverse<F> {
#[inline]
fn _shadow_map_id(&self, file_id: FileId) -> FileResult<PathBuf> {
self.path_for_id(file_id)
fn reset_shadow(&mut self) {
self.increment_revision(|this| this.vfs.reset_shadow())
}
#[inline]
fn shadow_paths(&self) -> Vec<Arc<Path>> {
self.vfs.shadow_paths()
}
#[inline]
fn reset_shadow(&mut self) {
self.increment_revision(|this| this.vfs.reset_shadow())
fn shadow_ids(&self) -> Vec<TypstFileId> {
self.vfs.shadow_ids()
}
#[inline]
@ -301,6 +292,19 @@ impl<F: CompilerFeat> ShadowApi for CompilerUniverse<F> {
Ok(())
})
}
#[inline]
fn map_shadow_by_id(&mut self, file_id: FileId, content: Bytes) -> FileResult<()> {
self.increment_revision(|this| this.vfs().map_shadow_by_id(file_id, content))
}
#[inline]
fn unmap_shadow_by_id(&mut self, file_id: FileId) -> FileResult<()> {
self.increment_revision(|this| {
this.vfs().remove_shadow_by_id(file_id);
Ok(())
})
}
}
impl<F: CompilerFeat> EntryReader for CompilerUniverse<F> {
@ -388,25 +392,19 @@ impl<F: CompilerFeat> CompilerWorld<F> {
}
/// Resolve the real path for a file id.
pub fn path_for_id(&self, id: FileId) -> Result<PathBuf, FileError> {
if id == *DETACHED_ENTRY {
return Ok(DETACHED_ENTRY.vpath().as_rooted_path().to_owned());
}
// Determine the root path relative to which the file path
// will be resolved.
let root = match id.package() {
Some(spec) => self.registry.resolve(spec)?,
None => self.entry.root().ok_or(FileError::Other(Some(eco_format!(
"cannot access directory without root: state: {:?}",
self.entry
))))?,
};
// Join the path to the root. If it tries to escape, deny
// access. Note: It can still escape via symlinks.
id.vpath().resolve(&root).ok_or(FileError::AccessDenied)
pub fn path_for_id(&self, id: FileId) -> Result<PathResolution, FileError> {
self.vfs.file_path(id)
}
/// Resolve the root of the workspace.
pub fn id_for_path(&self, path: &Path) -> Option<FileId> {
let root = self.entry.workspace_root()?;
Some(WorkspaceResolver::workspace_file(
Some(&root),
VirtualPath::new(path.strip_prefix(&root).ok()?),
))
}
/// Lookup a source file by id.
#[track_caller]
fn lookup(&self, id: FileId) -> Source {
@ -433,8 +431,8 @@ impl<F: CompilerFeat> CompilerWorld<F> {
impl<F: CompilerFeat> ShadowApi for CompilerWorld<F> {
#[inline]
fn _shadow_map_id(&self, file_id: FileId) -> FileResult<PathBuf> {
self.path_for_id(file_id)
fn shadow_ids(&self) -> Vec<TypstFileId> {
self.vfs.shadow_ids()
}
#[inline]
@ -457,6 +455,31 @@ impl<F: CompilerFeat> ShadowApi for CompilerWorld<F> {
self.vfs.remove_shadow(path);
Ok(())
}
#[inline]
fn map_shadow_by_id(&mut self, file_id: TypstFileId, content: Bytes) -> FileResult<()> {
self.vfs.map_shadow_by_id(file_id, content)
}
#[inline]
fn unmap_shadow_by_id(&mut self, file_id: TypstFileId) -> FileResult<()> {
self.vfs.remove_shadow_by_id(file_id);
Ok(())
}
}
impl<F: CompilerFeat> FsProvider for CompilerWorld<F> {
fn file_path(&self, fid: TypstFileId) -> FileResult<PathResolution> {
self.vfs.file_path(fid)
}
fn read(&self, fid: TypstFileId) -> FileResult<Bytes> {
self.vfs.read(fid)
}
fn is_file(&self, fid: TypstFileId) -> FileResult<bool> {
self.vfs.is_file(fid)
}
}
impl<F: CompilerFeat> World for CompilerWorld<F> {
@ -494,14 +517,12 @@ impl<F: CompilerFeat> World for CompilerWorld<F> {
return Ok(DETACH_SOURCE.clone());
}
let fid = self.vfs.file_id(&self.path_for_id(id)?);
self.source_db.source(id, fid, &self.vfs)
self.source_db.source(id, self)
}
/// Try to access the specified file.
fn file(&self, id: FileId) -> FileResult<Bytes> {
let fid = self.vfs.file_id(&self.path_for_id(id)?);
self.source_db.file(id, fid, &self.vfs)
self.source_db.file(id, self)
}
/// Get the current date.
@ -545,8 +566,8 @@ impl<F: CompilerFeat> EntryReader for CompilerWorld<F> {
impl<F: CompilerFeat> WorldDeps for CompilerWorld<F> {
#[inline]
fn iter_dependencies(&self, f: &mut dyn FnMut(ImmutPath)) {
self.source_db.iter_dependencies_dyn(&self.vfs, f)
fn iter_dependencies(&self, f: &mut dyn FnMut(TypstFileId)) {
self.source_db.iter_dependencies_dyn(f)
}
}
@ -563,24 +584,9 @@ impl<'a, F: CompilerFeat> codespan_reporting::files::Files<'a> for CompilerWorld
/// The user-facing name of a file.
fn name(&'a self, id: FileId) -> CodespanResult<Self::Name> {
let vpath = id.vpath();
Ok(if let Some(package) = id.package() {
format!("{package}{}", vpath.as_rooted_path().display())
} else {
match self.entry.root() {
Some(root) => {
// Try to express the path relative to the working directory.
vpath
.resolve(&root)
// differ from typst
// .and_then(|abs| pathdiff::diff_paths(&abs, self.workdir()))
.as_deref()
.unwrap_or_else(|| vpath.as_rootless_path())
.to_string_lossy()
.into()
}
None => vpath.as_rooted_path().display().to_string(),
}
Ok(match self.path_for_id(id) {
Ok(path) => path.as_path().display().to_string(),
Err(_) => format!("{id:?}"),
})
}

7
crates/tinymist/src/actor/typ_server.rs
View file

@ -12,6 +12,7 @@ use reflexo_typst::{
features::WITH_COMPILING_STATUS_FEATURE, typst::prelude::EcoVec, CompileEnv, CompileReport,
Compiler, ConsoleDiagReporter, FeatureSet, GenericExporter, TypstDocument,
};
use tinymist_project::vfs::FsProvider;
use tinymist_project::watch_deps;
use tokio::sync::{mpsc, oneshot};
use typst::diag::{SourceDiagnostic, SourceResult};
@ -548,7 +549,11 @@ impl<F: CompilerFeat + Send + Sync + 'static> CompileServerActor<F> {
// Notify the new file dependencies.
let mut deps = vec![];
world.iter_dependencies(&mut |dep| deps.push(dep.clone()));
world.iter_dependencies(&mut |dep| {
if let Ok(x) = world.file_path(dep).and_then(|e| e.to_err()) {
deps.push(x.into())
}
});
send(CompilerResponse::Notify(NotifyMessage::SyncDependency(
deps,
)));

4
crates/tinymist/src/cmd.rs
View file

@ -665,7 +665,7 @@ impl LanguageState {
let entry: StrResult<EntryState> = Ok(()).and_then(|_| {
let toml_id = tinymist_query::package::get_manifest_id(&info)?;
let toml_path = world.path_for_id(toml_id)?;
let toml_path = world.path_for_id(toml_id)?.as_path().to_owned();
let pkg_root = toml_path.parent().ok_or_else(|| {
eco_format!("cannot get package root (parent of {toml_path:?})")
})?;
@ -673,7 +673,7 @@ impl LanguageState {
let manifest = tinymist_query::package::get_manifest(world, toml_id)?;
let entry_point = toml_id.join(&manifest.package.entrypoint);
Ok(EntryState::new_rooted(pkg_root.into(), Some(entry_point)))
Ok(EntryState::new_rooted_by_id(pkg_root.into(), entry_point))
});
let entry = entry.map_err(|e| internal_error(e.to_string()))?;

9
crates/tinymist/src/init.rs
View file

@ -20,6 +20,7 @@ use tinymist_render::PeriscopeArgs;
use typst::foundations::IntoValue;
use typst::syntax::FileId;
use typst_shim::utils::{Deferred, LazyHash};
use vfs::WorkspaceResolver;
// todo: svelte-language-server responds to a Goto Definition request with
// LocationLink[] even if the client does not report the
@ -853,7 +854,7 @@ impl PathPattern {
let (root, main) = root.zip(main)?;
// Files in packages are not exported
if main.package().is_some() {
if WorkspaceResolver::is_package_file(main) {
return None;
}
// Files without a path are not exported
@ -1071,10 +1072,8 @@ mod tests {
#[test]
fn test_substitute_path() {
let root = Path::new("/root");
let entry = EntryState::new_rooted(
root.into(),
Some(FileId::new(None, VirtualPath::new("/dir1/dir2/file.txt"))),
);
let entry =
EntryState::new_rooted(root.into(), Some(VirtualPath::new("/dir1/dir2/file.txt")));
assert_eq!(
PathPattern::new("/substitute/$dir/$name").substitute(&entry),

8
crates/tinymist/src/resource/symbols.rs
View file

@ -3,6 +3,8 @@ use std::{collections::BTreeMap, path::Path, sync::Arc};
// use reflexo_typst::font::GlyphId;
use reflexo_typst::{vector::font::GlyphId, TypstDocument, TypstFont};
use sync_lsp::LspResult;
use typst::syntax::VirtualPath;
use typst::World;
use crate::world::{base::ShadowApi, EntryState, TaskInputs};
use crate::{actor::typ_client::WorldSnapFut, z_internal_error};
@ -978,16 +980,14 @@ impl LanguageState {
let font = {
let entry_path: Arc<Path> = Path::new("/._sym_.typ").into();
let new_entry = EntryState::new_rootless(entry_path.clone())
.ok_or_else(|| error_once!("cannot change entry"))
.map_err(z_internal_error)?;
let new_entry = EntryState::new_rootless(VirtualPath::new(&entry_path));
let mut forked = snap.world.task(TaskInputs {
entry: Some(new_entry),
..Default::default()
});
forked
.map_shadow(&entry_path, math_shaping_text.into_bytes().into())
.map_shadow_by_id(forked.main(), math_shaping_text.into_bytes().into())
.map_err(|e| error_once!("cannot map shadow", err: e))
.map_err(z_internal_error)?;

17
crates/tinymist/src/server.rs
View file

@ -13,7 +13,7 @@ use lsp_server::RequestId;
use lsp_types::request::{GotoDeclarationParams, WorkspaceConfiguration};
use lsp_types::*;
use once_cell::sync::OnceCell;
use reflexo_typst::{error::prelude::*, Bytes, Error, ImmutPath, Time};
use reflexo_typst::{error::prelude::*, Bytes, Error, ImmutPath};
use request::{RegisterCapability, UnregisterCapability};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value as JsonValue};
@ -280,7 +280,7 @@ impl LanguageState {
.iter()
.map(|(path, meta)| {
let content = meta.content.clone().text().as_bytes().into();
(path.clone(), FileResult::Ok((meta.mt, content)).into())
(path.clone(), FileResult::Ok(content).into())
})
.collect(),
)
@ -918,13 +918,11 @@ impl LanguageState {
/// Create a new source file.
pub fn create_source(&mut self, path: PathBuf, content: String) -> Result<(), Error> {
let now = Time::now();
let path: ImmutPath = path.into();
self.memory_changes.insert(
path.clone(),
MemoryFileMeta {
mt: now,
content: Source::detached(content.clone()),
},
);
@ -933,7 +931,7 @@ impl LanguageState {
log::info!("create source: {:?}", path);
// todo: is it safe to believe that the path is normalized?
let files = FileChangeSet::new_inserts(vec![(path, FileResult::Ok((now, content)).into())]);
let files = FileChangeSet::new_inserts(vec![(path, FileResult::Ok(content).into())]);
self.update_source(files)
}
@ -958,7 +956,6 @@ impl LanguageState {
content: Vec<TextDocumentContentChangeEvent>,
position_encoding: PositionEncoding,
) -> Result<(), Error> {
let now = Time::now();
let path: ImmutPath = path.into();
let meta = self
@ -980,9 +977,7 @@ impl LanguageState {
}
}
meta.mt = now;
let snapshot = FileResult::Ok((now, meta.content.text().as_bytes().into())).into();
let snapshot = FileResult::Ok(meta.content.text().as_bytes().into()).into();
let files = FileChangeSet::new_inserts(vec![(path.clone(), snapshot)]);
@ -1048,7 +1043,7 @@ impl LanguageState {
.map(|path| client.entry_resolver().resolve(Some(path.into())))
.or_else(|| {
let root = client.entry_resolver().root(None)?;
Some(EntryState::new_rooted(root, Some(*DETACHED_ENTRY)))
Some(EntryState::new_rooted_by_id(root, *DETACHED_ENTRY))
});
just_future(async move {
@ -1103,8 +1098,6 @@ impl LanguageState {
/// Metadata for a source file.
#[derive(Debug, Clone)]
pub struct MemoryFileMeta {
/// The last modified time.
pub mt: Time,
/// The content of the file.
pub content: Source,
}

5
crates/tinymist/src/tool/package/init.rs
View file

@ -100,13 +100,14 @@ fn scaffold_project(
}
}
let package_root = world.path_for_id(toml_id)?;
let package_root = world.path_for_id(toml_id)?.as_path().to_owned();
let package_root = package_root
.parent()
.ok_or_else(|| eco_format!("package root is not a directory (at {:?})", toml_id))?;
let template_dir = toml_id.join(tmpl_info.path.as_str());
let real_template_dir = world.path_for_id(template_dir)?;
// todo: template in memory
let real_template_dir = world.path_for_id(template_dir)?.to_err()?;
if !real_template_dir.exists() {
bail!(
"template directory does not exist (at {})",

22
crates/tinymist/src/tool/preview.rs
View file

@ -4,7 +4,6 @@ use std::num::NonZeroUsize;
use std::{collections::HashMap, net::SocketAddr, path::Path, sync::Arc};
use crate::world::vfs::notify::{FileChangeSet, MemoryEvent};
use crate::world::EntryReader;
use futures::{SinkExt, StreamExt, TryStreamExt};
use hyper::service::service_fn;
use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, HyperWebsocketStream};
@ -12,14 +11,14 @@ use hyper_util::rt::TokioIo;
use hyper_util::server::graceful::GracefulShutdown;
use lsp_types::notification::Notification;
use reflexo_typst::debug_loc::SourceSpanOffset;
use reflexo_typst::{error::prelude::*, Error, TypstDocument, TypstFileId};
use reflexo_typst::{error::prelude::*, Error, TypstDocument};
use serde::Serialize;
use serde_json::Value as JsonValue;
use sync_lsp::just_ok;
use tinymist_assets::TYPST_PREVIEW_HTML;
use tokio::sync::{mpsc, oneshot};
use typst::layout::{Frame, FrameItem, Point, Position};
use typst::syntax::{LinkedNode, Source, Span, SyntaxKind, VirtualPath};
use typst::syntax::{LinkedNode, Source, Span, SyntaxKind};
use typst::World;
pub use typst_preview::CompileStatus;
use typst_preview::{
@ -63,10 +62,7 @@ impl typst_preview::CompileView for PreviewCompileView {
let world = &self.snap.world;
let Location::Src(loc) = loc;
let filepath = Path::new(&loc.filepath);
let relative_path = filepath.strip_prefix(&world.workspace_root()?).ok()?;
let source_id = TypstFileId::new(None, VirtualPath::new(relative_path));
let source_id = world.id_for_path(Path::new(&loc.filepath))?;
let source = world.source(source_id).ok()?;
let cursor = source.line_column_to_byte(loc.pos.line, loc.pos.column)?;
@ -85,7 +81,6 @@ impl typst_preview::CompileView for PreviewCompileView {
let world = &self.snap.world;
let Location::Src(src_loc) = loc;
let path = Path::new(&src_loc.filepath).to_owned();
let line = src_loc.pos.line;
let column = src_loc.pos.column;
@ -93,14 +88,10 @@ impl typst_preview::CompileView for PreviewCompileView {
let Some(doc) = doc.as_deref() else {
return vec![];
};
let Some(root) = world.workspace_root() else {
return vec![];
};
let Some(relative_path) = path.strip_prefix(root).ok() else {
let Some(source_id) = world.id_for_path(Path::new(&src_loc.filepath)) else {
return vec![];
};
let source_id = TypstFileId::new(None, VirtualPath::new(relative_path));
let Some(source) = world.source(source_id).ok() else {
return vec![];
};
@ -123,7 +114,7 @@ impl typst_preview::CompileView for PreviewCompileView {
range.start += off;
}
}
let filepath = world.path_for_id(span.id()?).ok()?;
let filepath = world.path_for_id(span.id()?).ok()?.to_err().ok()?;
Some(DocToSrcJumpInfo {
filepath: filepath.to_string_lossy().to_string(),
start: resolve_off(&source, range.start),
@ -139,7 +130,6 @@ impl EditorServer for CompileHandler {
reset_shadow: bool,
) -> Result<(), Error> {
// todo: is it safe to believe that the path is normalized?
let now = std::time::SystemTime::now();
let files = FileChangeSet::new_inserts(
files
.files
@ -147,7 +137,7 @@ impl EditorServer for CompileHandler {
.map(|(path, content)| {
let content = content.as_bytes().into();
// todo: cloning PathBuf -> Arc<Path>
(path.into(), Ok((now, content)).into())
(path.into(), Ok(content).into())
})
.collect(),
);

14
crates/typlite/src/lib.rs
View file

@ -6,6 +6,7 @@ pub mod scopes;
pub mod value;
use core::fmt;
use std::path::Path;
use std::sync::Arc;
use std::{fmt::Write, sync::LazyLock};
@ -13,6 +14,7 @@ pub use error::*;
use base64::Engine;
use scopes::Scopes;
use tinymist_project::vfs::WorkspaceResolver;
use tinymist_project::{base::ShadowApi, EntryReader, LspWorld};
use tinymist_std::path::unix_slash;
use typst::foundations::IntoValue;
@ -28,7 +30,7 @@ use value::{Args, Value};
use ecow::{eco_format, EcoString};
use typst_syntax::{
ast::{self, AstNode},
FileId, Source, SyntaxKind, SyntaxNode, VirtualPath,
FileId, Source, SyntaxKind, SyntaxNode,
};
pub use typst_syntax as syntax;
@ -446,14 +448,14 @@ impl TypliteWorker {
let main = Bytes::from(code.as_bytes().to_owned());
// let world = LiteWorld::new(main);
let main_id = FileId::new(None, VirtualPath::new("__render__.typ"));
let entry = self.world.entry_state().select_in_workspace(main_id);
let path = Path::new("__render__.typ");
let entry = self.world.entry_state().select_in_workspace(path);
let mut world = self.world.task(tinymist_project::TaskInputs {
entry: Some(entry),
inputs,
});
world.source_db.take_state();
world.map_shadow_by_id(main_id, main).unwrap();
world.map_shadow_by_id(world.main(), main).unwrap();
let document = typst::compile(&world).output;
let document = document.map_err(|diagnostics| {
@ -461,10 +463,10 @@ impl TypliteWorker {
let _ = write!(err, "compiling node: ");
let write_span = |span: typst_syntax::Span, err: &mut String| {
let file = span.id().map(|id| match id.package() {
Some(package) => {
Some(package) if WorkspaceResolver::is_package_file(id) => {
format!("{package}:{}", unix_slash(id.vpath().as_rooted_path()))
}
None => unix_slash(id.vpath().as_rooted_path()),
Some(_) | None => unix_slash(id.vpath().as_rooted_path()),
});
let range = world.range(span);
match (file, range) {

5
crates/typlite/src/tests.rs
View file

@ -22,14 +22,15 @@ fn conv_(s: &str, for_docs: bool) -> EcoString {
let cwd = std::env::current_dir().unwrap();
let main = Source::detached(s);
let mut universe = LspUniverseBuilder::build(
EntryState::new_rooted(cwd.as_path().into(), Some(main.id())),
EntryState::new_rooted_by_id(cwd.as_path().into(), main.id()),
Default::default(),
FONT_RESOLVER.clone(),
Default::default(),
)
.unwrap();
let main_id = universe.main_id().unwrap();
universe
.map_shadow_by_id(main.id(), Bytes::from(main.text().as_bytes().to_owned()))
.map_shadow_by_id(main_id, Bytes::from(main.text().as_bytes().to_owned()))
.unwrap();
let world = universe.snapshot();