From 2d2857e6f31fc44985fcd5fe4dc3f7c096972181 Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin <35292584+Myriad-Dreamin@users.noreply.github.com> Date: Sat, 16 Mar 2024 02:54:56 +0800 Subject: [PATCH] docs: refactor and documenting analyzer code (#44) * dev: refactor and documenting analyzer code * dev: documenting some lsp api --- crates/tinymist-query/src/analysis.rs | 19 +-- crates/tinymist-query/src/analysis/def_use.rs | 137 ++++++------------ crates/tinymist-query/src/analysis/global.rs | 127 +++++++++------- .../src/analysis/track_values.rs | 2 + crates/tinymist-query/src/code_lens.rs | 5 + crates/tinymist-query/src/diagnostics.rs | 2 + crates/tinymist-query/src/document_symbol.rs | 2 +- .../snaps/def_use@import_alias_both.typ.snap | 5 +- .../snaps/def_use@import_ident_alias.typ.snap | 5 +- crates/tinymist-query/src/folding_range.rs | 2 +- crates/tinymist-query/src/goto_declaration.rs | 6 +- crates/tinymist-query/src/goto_definition.rs | 53 ++++--- crates/tinymist-query/src/lib.rs | 8 + crates/tinymist-query/src/prepare_rename.rs | 39 +++-- crates/tinymist-query/src/references.rs | 34 +++-- crates/tinymist-query/src/rename.rs | 32 ++-- crates/tinymist-query/src/symbol.rs | 2 +- .../src/{analysis => syntax}/import.rs | 3 + .../{analysis => syntax}/lexical_hierarchy.rs | 20 +-- .../src/{analysis => syntax}/matcher.rs | 0 crates/tinymist-query/src/syntax/mod.rs | 87 +++++++++++ .../src/{analysis => syntax}/module.rs | 39 ++++- crates/tinymist-query/src/tests.rs | 2 +- crates/tinymist/src/actor/typst.rs | 9 +- 24 files changed, 381 insertions(+), 259 deletions(-) rename crates/tinymist-query/src/{analysis => syntax}/import.rs (95%) rename crates/tinymist-query/src/{analysis => syntax}/lexical_hierarchy.rs (98%) rename crates/tinymist-query/src/{analysis => syntax}/matcher.rs (100%) create mode 100644 crates/tinymist-query/src/syntax/mod.rs rename crates/tinymist-query/src/{analysis => syntax}/module.rs (58%) diff --git a/crates/tinymist-query/src/analysis.rs b/crates/tinymist-query/src/analysis.rs index a8d31c45..71f16290 100644 --- a/crates/tinymist-query/src/analysis.rs +++ b/crates/tinymist-query/src/analysis.rs @@ -1,13 +1,5 @@ pub mod def_use; pub use def_use::*; -pub mod import; -pub use import::*; -pub mod lexical_hierarchy; -pub(crate) use lexical_hierarchy::*; -pub mod matcher; -pub use matcher::*; -pub mod module; -pub use module::*; pub mod track_values; pub use track_values::*; @@ -20,8 +12,8 @@ mod module_tests { use typst_ts_core::path::unix_slash; use typst_ts_core::typst::prelude::EcoVec; - use crate::analysis::module::*; use crate::prelude::*; + use crate::syntax::module::*; use crate::tests::*; #[test] @@ -71,12 +63,11 @@ mod module_tests { #[cfg(test)] mod lexical_hierarchy_tests { - use def_use::get_def_use; use def_use::DefUseSnapshot; use crate::analysis::def_use; - use crate::analysis::lexical_hierarchy; use crate::prelude::*; + use crate::syntax::lexical_hierarchy; use crate::tests::*; #[test] @@ -96,10 +87,10 @@ mod lexical_hierarchy_tests { #[test] fn test_def_use() { fn def_use(set: &str) { - snapshot_testing(set, &|world, path| { - let source = get_suitable_source_in_workspace(world, &path).unwrap(); + snapshot_testing2(set, &|ctx, path| { + let source = ctx.source_by_path(&path).unwrap(); - let result = get_def_use(&mut AnalysisContext::new(world), source); + let result = ctx.def_use(source); let result = result.as_deref().map(DefUseSnapshot); assert_snapshot!(JsonRepr::new_redacted(result, &REDACT_LOC)); diff --git a/crates/tinymist-query/src/analysis/def_use.rs b/crates/tinymist-query/src/analysis/def_use.rs index 8035ffae..0af602cf 100644 --- a/crates/tinymist-query/src/analysis/def_use.rs +++ b/crates/tinymist-query/src/analysis/def_use.rs @@ -1,4 +1,5 @@ -use core::fmt; +//! Static analysis for def-use relations. + use std::{ collections::HashMap, ops::{Deref, Range}, @@ -10,106 +11,77 @@ use serde::Serialize; use typst::syntax::Source; use typst_ts_core::{path::unix_slash, TypstFileId}; -use crate::{adt::snapshot_map::SnapshotMap, analysis::find_source_by_import_path}; - -use super::{ - get_lexical_hierarchy, AnalysisContext, LexicalHierarchy, LexicalKind, LexicalScopeKind, - LexicalVarKind, ModSrc, SearchCtx, +use super::SearchCtx; +use crate::syntax::{ + find_source_by_import_path, get_lexical_hierarchy, IdentRef, LexicalHierarchy, LexicalKind, + LexicalScopeKind, LexicalVarKind, ModSrc, }; +use crate::{adt::snapshot_map::SnapshotMap, syntax::LexicalModKind}; pub use typst_ts_core::vector::ir::DefId; +/// The type namespace of def-use relations +/// +/// The symbols from different namespaces are not visible to each other. enum Ns { + /// Def-use for labels Label, + /// Def-use for values Value, } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct IdentRef { - pub name: String, - pub range: Range, -} - -impl PartialOrd for IdentRef { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for IdentRef { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.name - .cmp(&other.name) - .then_with(|| self.range.start.cmp(&other.range.start)) - } -} - -impl fmt::Display for IdentRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}@{:?}", self.name, self.range) - } -} - -impl Serialize for IdentRef { - fn serialize(&self, serializer: S) -> Result { - let s = self.to_string(); - serializer.serialize_str(&s) - } -} - +/// A flat and transient reference to some symbol in a source file. +/// +/// See [`IdentRef`] for definition of a "transient" reference. #[derive(Serialize, Clone)] pub struct IdentDef { + /// The name of the symbol. pub name: String, + /// The kind of the symbol. pub kind: LexicalKind, + /// The byte range of the symbol in the source file. pub range: Range, } type ExternalRefMap = HashMap<(TypstFileId, Option), Vec<(Option, IdentRef)>>; +/// The def-use information of a source file. #[derive(Default)] pub struct DefUseInfo { ident_defs: indexmap::IndexMap<(TypstFileId, IdentRef), IdentDef>, external_refs: ExternalRefMap, ident_refs: HashMap, - redefine_current: Option, - ident_redefines: HashMap, undefined_refs: Vec, exports_refs: Vec, - pub exports_defs: HashMap, + exports_defs: HashMap, } impl DefUseInfo { + /// Get the definition id of a symbol by its name reference. pub fn get_ref(&self, ident: &IdentRef) -> Option { self.ident_refs.get(ident).copied() } + /// Get the definition of a symbol by its unique id. pub fn get_def_by_id(&self, id: DefId) -> Option<(TypstFileId, &IdentDef)> { let ((fid, _), def) = self.ident_defs.get_index(id.0 as usize)?; Some((*fid, def)) } + /// Get the definition of a symbol by its name reference. pub fn get_def(&self, fid: TypstFileId, ident: &IdentRef) -> Option<(DefId, &IdentDef)> { - let (id, _, def) = self - .ident_defs - .get_full(&(fid, ident.clone())) - .or_else(|| { - if self.redefine_current == Some(fid) { - let def_id = self.ident_redefines.get(ident)?; - let kv = self.ident_defs.get_index(def_id.0 as usize)?; - Some((def_id.0 as usize, kv.0, kv.1)) - } else { - None - } - })?; + let (id, _, def) = self.ident_defs.get_full(&(fid, ident.clone()))?; Some((DefId(id as u64), def)) } + /// Get the references of a symbol by its unique id. pub fn get_refs(&self, id: DefId) -> impl Iterator { self.ident_refs .iter() .filter_map(move |(k, v)| if *v == id { Some(k) } else { None }) } + /// Get external references of a symbol by its name reference. pub fn get_external_refs( &self, ext_id: TypstFileId, @@ -121,16 +93,13 @@ impl DefUseInfo { .flatten() } + /// Check if a symbol is exported. pub fn is_exported(&self, id: DefId) -> bool { self.exports_refs.contains(&id) } } -pub fn get_def_use(ctx: &mut AnalysisContext, source: Source) -> Option> { - get_def_use_inner(&mut ctx.fork_for_search(), source) -} - -fn get_def_use_inner(ctx: &mut SearchCtx, source: Source) -> Option> { +pub(super) fn get_def_use_inner(ctx: &mut SearchCtx, source: Source) -> Option> { let current_id = source.id(); ctx.ctx.get_mut(current_id); let c = ctx.ctx.get(current_id).unwrap(); @@ -155,7 +124,6 @@ fn get_def_use_inner(ctx: &mut SearchCtx, source: Source) -> Option DefUseCollector<'a, 'b, 'w> { | LexicalKind::Var(LexicalVarKind::Variable) => { self.insert(Ns::Value, e); } - LexicalKind::Mod(super::LexicalModKind::PathVar) - | LexicalKind::Mod(super::LexicalModKind::ModuleAlias) => { - self.insert_module(Ns::Value, e) - } - LexicalKind::Mod(super::LexicalModKind::Ident) => { - match self.import_name(&e.info.name) { - Some(()) => { - self.insert_ref(Ns::Value, e); - self.insert_redef(e); - } - None => { - let def_id = self.insert(Ns::Value, e); - self.insert_extern( - e.info.name.clone(), - e.info.range.clone(), - Some(def_id), - ); - } + LexicalKind::Mod(LexicalModKind::PathVar) + | LexicalKind::Mod(LexicalModKind::ModuleAlias) => self.insert_module(Ns::Value, e), + LexicalKind::Mod(LexicalModKind::Ident) => match self.import_name(&e.info.name) { + Some(()) => { + self.insert_ref(Ns::Value, e); } - } - LexicalKind::Mod(super::LexicalModKind::Alias { target }) => { + None => { + let def_id = self.insert(Ns::Value, e); + self.insert_extern(e.info.name.clone(), e.info.range.clone(), Some(def_id)); + } + }, + LexicalKind::Mod(LexicalModKind::Alias { target }) => { match self.import_name(&target.name) { Some(()) => { self.insert_ident_ref( @@ -288,7 +247,7 @@ impl<'a, 'b, 'w> DefUseCollector<'a, 'b, 'w> { self.enter(|this| this.scan(e.as_slice()))?; } } - LexicalKind::Mod(super::LexicalModKind::Module(p)) => { + LexicalKind::Mod(LexicalModKind::Module(p)) => { match p { ModSrc::Expr(_) => {} ModSrc::Path(p) => { @@ -308,7 +267,7 @@ impl<'a, 'b, 'w> DefUseCollector<'a, 'b, 'w> { self.ext_src = None; } - LexicalKind::Mod(super::LexicalModKind::Star) => { + LexicalKind::Mod(LexicalModKind::Star) => { if let Some(source) = &self.ext_src { info!("diving source for def use: {:?}", source.id()); let (_, external_info) = @@ -399,21 +358,9 @@ impl<'a, 'b, 'w> DefUseCollector<'a, 'b, 'w> { }, ); } - - fn insert_redef(&mut self, e: &LexicalHierarchy) { - let snap = &mut self.id_scope; - - let id_ref = IdentRef { - name: e.info.name.clone(), - range: e.info.range.clone(), - }; - - if let Some(id) = snap.get(&e.info.name) { - self.info.ident_redefines.insert(id_ref, *id); - } - } } +/// A snapshot of the def-use information for testing. pub struct DefUseSnapshot<'a>(pub &'a DefUseInfo); impl<'a> Serialize for DefUseSnapshot<'a> { diff --git a/crates/tinymist-query/src/analysis/global.rs b/crates/tinymist-query/src/analysis/global.rs index cc25355c..435893eb 100644 --- a/crates/tinymist-query/src/analysis/global.rs +++ b/crates/tinymist-query/src/analysis/global.rs @@ -13,25 +13,36 @@ use typst::{ use typst_ts_compiler::{service::WorkspaceProvider, TypstSystemWorld}; use typst_ts_core::{cow_mut::CowMut, ImmutPath, TypstFileId}; -use super::{construct_module_dependencies, DefUseInfo, ModuleDependency}; +use super::{get_def_use_inner, DefUseInfo}; +use crate::{ + lsp_to_typst, + syntax::{construct_module_dependencies, scan_workspace_files, ModuleDependency}, + typst_to_lsp, LspPosition, LspRange, PositionEncoding, TypstRange, +}; +/// A cache for module-level analysis results of a module. +/// +/// You should not holds across requests, because source code may change. pub struct ModuleAnalysisCache { source: OnceCell>, def_use: OnceCell>>, } impl ModuleAnalysisCache { + /// Get the source of a file. pub fn source(&self, ctx: &AnalysisContext, file_id: TypstFileId) -> FileResult { self.source .get_or_init(|| ctx.world.source(file_id)) .clone() } + /// Try to get the def-use information of a file. pub fn def_use(&self) -> Option> { self.def_use.get().cloned().flatten() } - pub fn compute_def_use( + /// Compute the def-use information of a file. + pub(crate) fn compute_def_use( &self, f: impl FnOnce() -> Option>, ) -> Option> { @@ -39,28 +50,40 @@ impl ModuleAnalysisCache { } } +/// The analysis data holds globally. pub struct Analysis { + /// The root of the workspace. + /// This means that the analysis result won't be valid if the root directory + /// changes. pub root: ImmutPath, + /// The position encoding for the workspace. + position_encoding: PositionEncoding, } +/// A cache for all level of analysis results of a module. pub struct AnalysisCaches { modules: HashMap, root_files: OnceCell>, module_deps: OnceCell>, } +/// The context for analyzers. pub struct AnalysisContext<'a> { + /// The world surface for Typst compiler pub world: &'a TypstSystemWorld, + /// The analysis data pub analysis: CowMut<'a, Analysis>, caches: AnalysisCaches, } impl<'w> AnalysisContext<'w> { - pub fn new(world: &'w TypstSystemWorld) -> Self { + /// Create a new analysis context. + pub fn new(world: &'w TypstSystemWorld, encoding: PositionEncoding) -> Self { Self { world, analysis: CowMut::Owned(Analysis { root: world.workspace_root(), + position_encoding: encoding, }), caches: AnalysisCaches { modules: HashMap::new(), @@ -75,10 +98,14 @@ impl<'w> AnalysisContext<'w> { self.caches.root_files.get_or_init(f) } + /// Get all the files in the workspace. pub fn files(&mut self) -> &Vec { - self.caches.root_files.get_or_init(|| self.search_files()) + self.caches + .root_files + .get_or_init(|| scan_workspace_files(&self.analysis.root)) } + /// Get the module dependencies of the workspace. pub fn module_dependencies(&mut self) -> &HashMap { if self.caches.module_deps.get().is_some() { return self.caches.module_deps.get().unwrap(); @@ -90,31 +117,13 @@ impl<'w> AnalysisContext<'w> { } } - pub fn fork_for_search<'s>(&'s mut self) -> SearchCtx<'s, 'w> { - SearchCtx { - ctx: self, - searched: Default::default(), - worklist: Default::default(), - } - } - - pub fn get_mut(&mut self, file_id: TypstFileId) -> &ModuleAnalysisCache { - self.caches.modules.entry(file_id).or_insert_with(|| { - let source = OnceCell::new(); - let def_use = OnceCell::new(); - ModuleAnalysisCache { source, def_use } - }) - } - - pub fn get(&self, file_id: TypstFileId) -> Option<&ModuleAnalysisCache> { - self.caches.modules.get(&file_id) - } - + /// Get the source of a file by file id. pub fn source_by_id(&mut self, id: TypstFileId) -> FileResult { self.get_mut(id); self.get(id).unwrap().source(self, id) } + /// Get the source of a file by file path. pub fn source_by_path(&mut self, p: &Path) -> FileResult { // todo: source in packages let relative_path = p.strip_prefix(&self.analysis.root).map_err(|_| { @@ -128,48 +137,63 @@ impl<'w> AnalysisContext<'w> { self.source_by_id(id) } - fn search_files(&self) -> Vec { - let root = self.analysis.root.clone(); + /// Get the module-level analysis cache of a file. + pub fn get(&self, file_id: TypstFileId) -> Option<&ModuleAnalysisCache> { + self.caches.modules.get(&file_id) + } - let mut res = vec![]; - for path in walkdir::WalkDir::new(&root).follow_links(false).into_iter() { - let Ok(de) = path else { - continue; - }; - if !de.file_type().is_file() { - continue; - } - if !de - .path() - .extension() - .is_some_and(|e| e == "typ" || e == "typc") - { - continue; - } + /// Get the module-level analysis cache of a file. + pub fn get_mut(&mut self, file_id: TypstFileId) -> &ModuleAnalysisCache { + self.caches.modules.entry(file_id).or_insert_with(|| { + let source = OnceCell::new(); + let def_use = OnceCell::new(); + ModuleAnalysisCache { source, def_use } + }) + } - let path = de.path(); - let relative_path = match path.strip_prefix(&root) { - Ok(p) => p, - Err(err) => { - log::warn!("failed to strip prefix, path: {path:?}, root: {root:?}: {err}"); - continue; - } - }; + /// Get the def-use information of a source file. + pub fn def_use(&mut self, source: Source) -> Option> { + get_def_use_inner(&mut self.fork_for_search(), source) + } - res.push(TypstFileId::new(None, VirtualPath::new(relative_path))); + /// Fork a new context for searching in the workspace. + pub fn fork_for_search<'s>(&'s mut self) -> SearchCtx<'s, 'w> { + SearchCtx { + ctx: self, + searched: Default::default(), + worklist: Default::default(), } + } - res + pub fn to_typst_pos(&self, position: LspPosition, src: &Source) -> Option { + lsp_to_typst::position(position, self.analysis.position_encoding, src) + } + + pub fn to_typst_range(&self, position: LspRange, src: &Source) -> Option { + lsp_to_typst::range(position, self.analysis.position_encoding, src) + } + + pub fn to_lsp_range(&self, position: TypstRange, src: &Source) -> LspRange { + typst_to_lsp::range(position, src, self.analysis.position_encoding) + } + + pub(crate) fn position_encoding(&self) -> PositionEncoding { + self.analysis.position_encoding } } +/// The context for searching in the workspace. pub struct SearchCtx<'b, 'w> { + /// The inner analysis context. pub ctx: &'b mut AnalysisContext<'w>, + /// The set of files that have been searched. pub searched: HashSet, + /// The files that need to be searched. pub worklist: Vec, } impl SearchCtx<'_, '_> { + /// Push a file to the worklist. pub fn push(&mut self, id: TypstFileId) -> bool { if self.searched.insert(id) { self.worklist.push(id); @@ -179,6 +203,7 @@ impl SearchCtx<'_, '_> { } } + /// Push the dependents of a file to the worklist. pub fn push_dependents(&mut self, id: TypstFileId) { let deps = self.ctx.module_dependencies().get(&id); let dependents = deps.map(|e| e.dependents.clone()).into_iter().flatten(); diff --git a/crates/tinymist-query/src/analysis/track_values.rs b/crates/tinymist-query/src/analysis/track_values.rs index c05fcb10..d70d9ead 100644 --- a/crates/tinymist-query/src/analysis/track_values.rs +++ b/crates/tinymist-query/src/analysis/track_values.rs @@ -1,3 +1,5 @@ +//! Dynamic analysis of an expression or import statement. + use comemo::Track; use typst::engine::{Engine, Route}; use typst::eval::{Tracer, Vm}; diff --git a/crates/tinymist-query/src/code_lens.rs b/crates/tinymist-query/src/code_lens.rs index 027d84d9..46446c65 100644 --- a/crates/tinymist-query/src/code_lens.rs +++ b/crates/tinymist-query/src/code_lens.rs @@ -2,8 +2,13 @@ use lsp_types::Command; use crate::prelude::*; +/// The [`textDocument/codeLens`] request is sent from the client to the server +/// to compute code lenses for a given text document. +/// +/// [`textDocument/codeLens`]: https://microsoft.github.io/language-server-protocol/specification#textDocument_codeLens #[derive(Debug, Clone)] pub struct CodeLensRequest { + /// The path of the document to request for. pub path: PathBuf, } diff --git a/crates/tinymist-query/src/diagnostics.rs b/crates/tinymist-query/src/diagnostics.rs index bdd8953f..588c9b18 100644 --- a/crates/tinymist-query/src/diagnostics.rs +++ b/crates/tinymist-query/src/diagnostics.rs @@ -1,7 +1,9 @@ use crate::prelude::*; +/// Stores diagnostics for files. pub type DiagnosticsMap = HashMap>; +/// Converts a list of Typst diagnostics to LSP diagnostics. pub fn convert_diagnostics<'a>( project: &TypstSystemWorld, errors: impl IntoIterator, diff --git a/crates/tinymist-query/src/document_symbol.rs b/crates/tinymist-query/src/document_symbol.rs index fae7cc88..badc70e7 100644 --- a/crates/tinymist-query/src/document_symbol.rs +++ b/crates/tinymist-query/src/document_symbol.rs @@ -1,6 +1,6 @@ use crate::{ - analysis::{get_lexical_hierarchy, LexicalHierarchy, LexicalScopeKind}, prelude::*, + syntax::{get_lexical_hierarchy, LexicalHierarchy, LexicalScopeKind}, }; #[derive(Debug, Clone)] diff --git a/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_alias_both.typ.snap b/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_alias_both.typ.snap index acc52554..da0bcbab 100644 --- a/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_alias_both.typ.snap +++ b/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_alias_both.typ.snap @@ -32,10 +32,7 @@ input_file: crates/tinymist-query/src/fixtures/def_use/import_alias_both.typ "kind": { "Mod": { "Alias": { - "target": { - "name": "x", - "range": "54:55" - } + "target": "x@54..55" } } }, diff --git a/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_ident_alias.typ.snap b/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_ident_alias.typ.snap index a5522fec..d61e92b8 100644 --- a/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_ident_alias.typ.snap +++ b/crates/tinymist-query/src/fixtures/def_use/snaps/def_use@import_ident_alias.typ.snap @@ -22,10 +22,7 @@ input_file: crates/tinymist-query/src/fixtures/def_use/import_ident_alias.typ "kind": { "Mod": { "Alias": { - "target": { - "name": "x", - "range": "47:48" - } + "target": "x@47..48" } } }, diff --git a/crates/tinymist-query/src/folding_range.rs b/crates/tinymist-query/src/folding_range.rs index b0f39462..50ceb016 100644 --- a/crates/tinymist-query/src/folding_range.rs +++ b/crates/tinymist-query/src/folding_range.rs @@ -1,6 +1,6 @@ use crate::{ - analysis::{get_lexical_hierarchy, LexicalHierarchy, LexicalKind, LexicalScopeKind}, prelude::*, + syntax::{get_lexical_hierarchy, LexicalHierarchy, LexicalKind, LexicalScopeKind}, }; #[derive(Debug, Clone)] diff --git a/crates/tinymist-query/src/goto_declaration.rs b/crates/tinymist-query/src/goto_declaration.rs index 82627165..9a087c09 100644 --- a/crates/tinymist-query/src/goto_declaration.rs +++ b/crates/tinymist-query/src/goto_declaration.rs @@ -4,8 +4,8 @@ use log::debug; use lsp_types::LocationLink; use crate::{ - analysis::{get_def_use, get_deref_target, DerefTarget}, prelude::*, + syntax::{get_deref_target, DerefTarget}, }; #[derive(Debug, Clone)] @@ -20,7 +20,7 @@ impl GotoDeclarationRequest { world: &TypstSystemWorld, position_encoding: PositionEncoding, ) -> Option { - let mut ctx = AnalysisContext::new(world); + let mut ctx = AnalysisContext::new(world, position_encoding); let source = get_suitable_source_in_workspace(world, &self.path).ok()?; let offset = lsp_to_typst::position(self.position, position_encoding, &source)?; let cursor = offset + 1; @@ -34,7 +34,7 @@ impl GotoDeclarationRequest { let origin_selection_range = typst_to_lsp::range(use_site.range(), &source, position_encoding); - let def_use = get_def_use(&mut ctx, source.clone())?; + let def_use = ctx.def_use(source.clone())?; let ref_spans = find_declarations(w, def_use, deref_target)?; let mut links = vec![]; diff --git a/crates/tinymist-query/src/goto_definition.rs b/crates/tinymist-query/src/goto_definition.rs index d119c2ab..026d4af7 100644 --- a/crates/tinymist-query/src/goto_definition.rs +++ b/crates/tinymist-query/src/goto_definition.rs @@ -1,37 +1,47 @@ use std::ops::Range; use log::debug; -use typst::{ - foundations::Value, - syntax::{ - ast::{self}, - LinkedNode, Source, - }, -}; +use typst::foundations::Value; use typst_ts_core::TypstFileId; use crate::{ - analysis::{ - find_source_by_import, get_def_use, get_deref_target, DerefTarget, IdentRef, LexicalKind, + prelude::*, + syntax::{ + find_source_by_import, get_deref_target, DerefTarget, IdentRef, LexicalKind, LexicalModKind, LexicalVarKind, }, - prelude::*, + SyntaxRequest, }; +/// The [`textDocument/definition`] request asks the server for the definition +/// location of a symbol at a given text document position. +/// +/// [`textDocument/definition`]: https://microsoft.github.io/language-server-protocol/specification#textDocument_definition +/// +/// # Compatibility +/// +/// The [`GotoDefinitionResponse::Link`](lsp_types::GotoDefinitionResponse::Link) return value +/// was introduced in specification version 3.14.0 and requires client-side +/// support in order to be used. It can be returned if the client set the +/// following field to `true` in the [`initialize`](Self::initialize) method: +/// +/// ```text +/// InitializeParams::capabilities::text_document::definition::link_support +/// ``` #[derive(Debug, Clone)] pub struct GotoDefinitionRequest { + /// The path of the document to request for. pub path: PathBuf, + /// The source code position to request for. pub position: LspPosition, } -impl GotoDefinitionRequest { - pub fn request( - self, - ctx: &mut AnalysisContext, - position_encoding: PositionEncoding, - ) -> Option { +impl SyntaxRequest for GotoDefinitionRequest { + type Response = GotoDefinitionResponse; + + fn request(self, ctx: &mut AnalysisContext) -> Option { let source = ctx.source_by_path(&self.path).ok()?; - let offset = lsp_to_typst::position(self.position, position_encoding, &source)?; + let offset = ctx.to_typst_pos(self.position, &source)?; let cursor = offset + 1; let ast_node = LinkedNode::new(source.root()).leaf_at(cursor)?; @@ -39,8 +49,7 @@ impl GotoDefinitionRequest { let deref_target = get_deref_target(ast_node)?; let use_site = deref_target.node().clone(); - let origin_selection_range = - typst_to_lsp::range(use_site.range(), &source, position_encoding); + let origin_selection_range = ctx.to_lsp_range(use_site.range(), &source); let def = find_definition(ctx, source.clone(), deref_target)?; @@ -48,7 +57,7 @@ impl GotoDefinitionRequest { let uri = Url::from_file_path(span_path).ok()?; let span_source = ctx.source_by_id(def.fid).ok()?; - let range = typst_to_lsp::range(def.def_range, &span_source, position_encoding); + let range = ctx.to_lsp_range(def.def_range, &span_source); let res = Some(GotoDefinitionResponse::Link(vec![LocationLink { origin_selection_range: Some(origin_selection_range), @@ -98,7 +107,7 @@ pub(crate) fn find_definition( }; // syntatic definition - let def_use = get_def_use(ctx, source)?; + let def_use = ctx.def_use(source)?; let ident_ref = match use_site.cast::()? { ast::Expr::Ident(e) => IdentRef { name: e.get().to_string(), @@ -220,7 +229,7 @@ mod tests { position: find_test_position(&source), }; - let result = request.request(world, PositionEncoding::Utf16); + let result = request.request(world); assert_snapshot!(JsonRepr::new_redacted(result, &REDACT_LOC)); }); } diff --git a/crates/tinymist-query/src/lib.rs b/crates/tinymist-query/src/lib.rs index 02113447..5182b596 100644 --- a/crates/tinymist-query/src/lib.rs +++ b/crates/tinymist-query/src/lib.rs @@ -1,10 +1,12 @@ mod adt; pub mod analysis; +pub mod syntax; pub(crate) mod diagnostics; use std::sync::Arc; +pub use analysis::AnalysisContext; use typst_ts_core::TypstDocument; pub use diagnostics::*; @@ -54,6 +56,12 @@ pub struct VersionedDocument { pub document: Arc, } +pub trait SyntaxRequest { + type Response; + + fn request(self, ctx: &mut AnalysisContext) -> Option; +} + mod polymorphic { use super::prelude::*; use super::*; diff --git a/crates/tinymist-query/src/prepare_rename.rs b/crates/tinymist-query/src/prepare_rename.rs index cf378dbd..87c1cb7c 100644 --- a/crates/tinymist-query/src/prepare_rename.rs +++ b/crates/tinymist-query/src/prepare_rename.rs @@ -1,28 +1,38 @@ -use crate::{analysis::get_deref_target, find_definition, prelude::*, DefinitionLink}; +use crate::{find_definition, prelude::*, syntax::get_deref_target, DefinitionLink, SyntaxRequest}; use log::debug; +/// The [`textDocument/prepareRename`] request is sent from the client to the +/// server to setup and test the validity of a rename operation at a given +/// location. +/// +/// [`textDocument/prepareRename`]: https://microsoft.github.io/language-server-protocol/specification#textDocument_prepareRename +/// +/// # Compatibility +/// +/// This request was introduced in specification version 3.12.0. +/// +/// See . +/// The prepareRename feature is sent before a rename request. If the user +/// is trying to rename a symbol that should not be renamed (inside a +/// string or comment, on a builtin identifier, etc.), VSCode won't even +/// show the rename pop-up. #[derive(Debug, Clone)] pub struct PrepareRenameRequest { + /// The path of the document to request for. pub path: PathBuf, + /// The source code position to request for. pub position: LspPosition, } // todo: rename alias // todo: rename import path? -impl PrepareRenameRequest { - /// See . - /// The prepareRename feature is sent before a rename request. If the user - /// is trying to rename a symbol that should not be renamed (inside a - /// string or comment, on a builtin identifier, etc.), VSCode won't even - /// show the rename pop-up. - pub fn request( - self, - ctx: &mut AnalysisContext, - position_encoding: PositionEncoding, - ) -> Option { +impl SyntaxRequest for PrepareRenameRequest { + type Response = PrepareRenameResponse; + + fn request(self, ctx: &mut AnalysisContext) -> Option { let source = ctx.source_by_path(&self.path).ok()?; - let offset = lsp_to_typst::position(self.position, position_encoding, &source)?; + let offset = ctx.to_typst_pos(self.position, &source)?; let cursor = offset + 1; let ast_node = LinkedNode::new(source.root()).leaf_at(cursor)?; @@ -30,8 +40,7 @@ impl PrepareRenameRequest { let deref_target = get_deref_target(ast_node)?; let use_site = deref_target.node().clone(); - let origin_selection_range = - typst_to_lsp::range(use_site.range(), &source, position_encoding); + let origin_selection_range = ctx.to_lsp_range(use_site.range(), &source); let lnk = find_definition(ctx, source.clone(), deref_target)?; validate_renaming_definition(&lnk)?; diff --git a/crates/tinymist-query/src/references.rs b/crates/tinymist-query/src/references.rs index 4f4df3aa..8c34ee62 100644 --- a/crates/tinymist-query/src/references.rs +++ b/crates/tinymist-query/src/references.rs @@ -2,32 +2,38 @@ use log::debug; use typst_ts_core::vector::ir::DefId; use crate::{ - analysis::{get_def_use, get_deref_target, DerefTarget, IdentRef}, prelude::*, + syntax::{get_deref_target, DerefTarget, IdentRef}, + SyntaxRequest, }; +/// The [`textDocument/references`] request is sent from the client to the +/// server to resolve project-wide references for the symbol denoted by the +/// given text document position. +/// +/// [`textDocument/references`]: https://microsoft.github.io/language-server-protocol/specification#textDocument_references #[derive(Debug, Clone)] pub struct ReferencesRequest { + /// The path of the document to request for. pub path: PathBuf, + /// The source code position to request for. pub position: LspPosition, } -impl ReferencesRequest { - pub fn request( - self, - ctx: &mut AnalysisContext, - position_encoding: PositionEncoding, - ) -> Option> { +impl SyntaxRequest for ReferencesRequest { + type Response = Vec; + + fn request(self, ctx: &mut AnalysisContext) -> Option { let source = ctx.source_by_path(&self.path).ok()?; - let offset = lsp_to_typst::position(self.position, position_encoding, &source)?; + let offset = ctx.to_typst_pos(self.position, &source)?; let cursor = offset + 1; let ast_node = LinkedNode::new(source.root()).leaf_at(cursor)?; debug!("ast_node: {ast_node:?}", ast_node = ast_node); let deref_target = get_deref_target(ast_node)?; - let def_use = get_def_use(ctx, source.clone())?; - let locations = find_references(ctx, def_use, deref_target, position_encoding)?; + let def_use = ctx.def_use(source.clone())?; + let locations = find_references(ctx, def_use, deref_target, ctx.position_encoding())?; debug!("references: {locations:?}"); Some(locations) @@ -88,7 +94,7 @@ pub(crate) fn find_references( }; let def_source = ctx.source_by_id(def_fid).ok()?; - let root_def_use = get_def_use(ctx, def_source)?; + let root_def_use = ctx.def_use(def_source)?; let root_def_id = root_def_use.get_def(def_fid, &def_ident)?.0; find_references_root( @@ -132,9 +138,7 @@ pub(crate) fn find_references_root( ctx.push_dependents(def_fid); while let Some(ref_fid) = ctx.worklist.pop() { let ref_source = ctx.ctx.source_by_id(ref_fid).ok()?; - let def_use = get_def_use(ctx.ctx, ref_source.clone())?; - - log::info!("def_use for {ref_fid:?} => {:?}", def_use.exports_defs); + let def_use = ctx.ctx.def_use(ref_source.clone())?; let uri = ctx.ctx.world.path_for_id(ref_fid).ok()?; let uri = Url::from_file_path(uri).ok()?; @@ -180,7 +184,7 @@ mod tests { position: find_test_position(&source), }; - let result = request.request(world, PositionEncoding::Utf16); + let result = request.request(world); // sort let result = result.map(|mut e| { e.sort_by(|a, b| match a.range.start.cmp(&b.range.start) { diff --git a/crates/tinymist-query/src/rename.rs b/crates/tinymist-query/src/rename.rs index fae815c7..79e5256c 100644 --- a/crates/tinymist-query/src/rename.rs +++ b/crates/tinymist-query/src/rename.rs @@ -2,28 +2,32 @@ use log::debug; use lsp_types::TextEdit; use crate::{ - analysis::{get_def_use, get_deref_target}, - find_definition, find_references, - prelude::*, - validate_renaming_definition, + find_definition, find_references, prelude::*, syntax::get_deref_target, + validate_renaming_definition, SyntaxRequest, }; +/// The [`textDocument/rename`] request is sent from the client to the server to +/// ask the server to compute a workspace change so that the client can perform +/// a workspace-wide rename of a symbol. +/// +/// [`textDocument/rename`]: https://microsoft.github.io/language-server-protocol/specification#textDocument_rename #[derive(Debug, Clone)] pub struct RenameRequest { + /// The path of the document to request for. pub path: PathBuf, + /// The source code position to request for. pub position: LspPosition, + /// The new name to rename to. pub new_name: String, } -impl RenameRequest { - pub fn request( - self, - ctx: &mut AnalysisContext, - position_encoding: PositionEncoding, - ) -> Option { +impl SyntaxRequest for RenameRequest { + type Response = WorkspaceEdit; + + fn request(self, ctx: &mut AnalysisContext) -> Option { let source = ctx.source_by_path(&self.path).ok()?; - let offset = lsp_to_typst::position(self.position, position_encoding, &source)?; + let offset = ctx.to_typst_pos(self.position, &source)?; let cursor = offset + 1; let ast_node = LinkedNode::new(source.root()).leaf_at(cursor)?; @@ -35,8 +39,8 @@ impl RenameRequest { validate_renaming_definition(&lnk)?; - let def_use = get_def_use(ctx, source.clone())?; - let references = find_references(ctx, def_use, deref_target, position_encoding)?; + let def_use = ctx.def_use(source.clone())?; + let references = find_references(ctx, def_use, deref_target, ctx.position_encoding())?; let mut editions = HashMap::new(); @@ -53,7 +57,7 @@ impl RenameRequest { LspLocation { uri, - range: typst_to_lsp::range(range, &def_source, position_encoding), + range: ctx.to_lsp_range(range, &def_source), } }; diff --git a/crates/tinymist-query/src/symbol.rs b/crates/tinymist-query/src/symbol.rs index 8db8518f..75245672 100644 --- a/crates/tinymist-query/src/symbol.rs +++ b/crates/tinymist-query/src/symbol.rs @@ -1,8 +1,8 @@ use typst_ts_compiler::NotifyApi; use crate::{ - analysis::{get_lexical_hierarchy, LexicalHierarchy, LexicalScopeKind}, prelude::*, + syntax::{get_lexical_hierarchy, LexicalHierarchy, LexicalScopeKind}, }; #[derive(Debug, Clone)] diff --git a/crates/tinymist-query/src/analysis/import.rs b/crates/tinymist-query/src/syntax/import.rs similarity index 95% rename from crates/tinymist-query/src/analysis/import.rs rename to crates/tinymist-query/src/syntax/import.rs index 0bee79e7..467f9b04 100644 --- a/crates/tinymist-query/src/analysis/import.rs +++ b/crates/tinymist-query/src/syntax/import.rs @@ -6,6 +6,7 @@ use typst_ts_core::{typst::prelude::EcoVec, TypstFileId}; use crate::prelude::*; +/// Find a source instance by its import path. pub fn find_source_by_import_path( world: &dyn World, current: TypstFileId, @@ -27,6 +28,7 @@ pub fn find_source_by_import_path( world.source(id).ok() } +/// Find a source instance by its import node. pub fn find_source_by_import( world: &dyn World, current: TypstFileId, @@ -40,6 +42,7 @@ pub fn find_source_by_import( } } +/// Find all static imports in a source. #[comemo::memoize] pub fn find_imports(source: &Source) -> EcoVec { let root = LinkedNode::new(source.root()); diff --git a/crates/tinymist-query/src/analysis/lexical_hierarchy.rs b/crates/tinymist-query/src/syntax/lexical_hierarchy.rs similarity index 98% rename from crates/tinymist-query/src/analysis/lexical_hierarchy.rs rename to crates/tinymist-query/src/syntax/lexical_hierarchy.rs index 49a5b227..f3fd266f 100644 --- a/crates/tinymist-query/src/analysis/lexical_hierarchy.rs +++ b/crates/tinymist-query/src/syntax/lexical_hierarchy.rs @@ -16,6 +16,8 @@ use typst::{ }; use typst_ts_core::typst::prelude::{eco_vec, EcoVec}; +use super::IdentRef; + pub(crate) fn get_lexical_hierarchy( source: Source, g: LexicalScopeKind, @@ -46,17 +48,11 @@ pub(crate) fn get_lexical_hierarchy( res.map(|_| worker.stack.pop().unwrap().1) } -#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] -pub struct ImportAlias { - pub name: String, - pub range: Range, -} - #[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] pub enum ModSrc { /// `import cetz.draw ...` /// ^^^^^^^^^^^^^^^^^^^^ - Expr(Box), + Expr(Box), /// `import "" ...` /// ^^^^^^^^^^^^^ Path(Box), @@ -77,7 +73,7 @@ pub enum LexicalModKind { Ident, /// `import "foo": bar as baz` /// ^^^^^^^^^^ - Alias { target: Box }, + Alias { target: Box }, /// `import "foo": *` /// ^ Star, @@ -147,7 +143,7 @@ impl LexicalKind { LexicalKind::Mod(LexicalModKind::Star) } - fn module_expr(path: Box) -> LexicalKind { + fn module_expr(path: Box) -> LexicalKind { LexicalKind::Mod(LexicalModKind::Module(ModSrc::Expr(path))) } @@ -155,7 +151,7 @@ impl LexicalKind { LexicalKind::Mod(LexicalModKind::Module(ModSrc::Path(path))) } - fn module_import_alias(alias: ImportAlias) -> LexicalKind { + fn module_import_alias(alias: IdentRef) -> LexicalKind { LexicalKind::Mod(LexicalModKind::Alias { target: Box::new(alias), }) @@ -455,7 +451,7 @@ impl LexicalHierarchyWorker { self.push_leaf(LexicalInfo { name: origin_name.get().to_string(), - kind: LexicalKind::module_import_alias(ImportAlias { + kind: LexicalKind::module_import_alias(IdentRef { name: target_name.get().to_string(), range: target_name_node.range(), }), @@ -574,7 +570,7 @@ impl LexicalHierarchyWorker { let e = node .find(src.span()) .ok_or_else(|| anyhow!("find expression failed: {:?}", src))?; - let e = ImportAlias { + let e = IdentRef { name: String::new(), range: e.range(), }; diff --git a/crates/tinymist-query/src/analysis/matcher.rs b/crates/tinymist-query/src/syntax/matcher.rs similarity index 100% rename from crates/tinymist-query/src/analysis/matcher.rs rename to crates/tinymist-query/src/syntax/matcher.rs diff --git a/crates/tinymist-query/src/syntax/mod.rs b/crates/tinymist-query/src/syntax/mod.rs new file mode 100644 index 00000000..63ef6367 --- /dev/null +++ b/crates/tinymist-query/src/syntax/mod.rs @@ -0,0 +1,87 @@ +pub mod import; +pub use import::*; +pub mod lexical_hierarchy; +pub(crate) use lexical_hierarchy::*; +pub mod matcher; +pub use matcher::*; +pub mod module; +pub use module::*; + +use core::fmt; +use std::ops::Range; + +use serde::{Deserialize, Serialize}; + +/// A flat and transient reference to some symbol in a source file. +/// +/// It is transient because it is not guaranteed to be valid after the source +/// file is modified. +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct IdentRef { + /// The name of the symbol. + pub name: String, + /// The byte range of the symbol in the source file. + pub range: Range, +} + +impl PartialOrd for IdentRef { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for IdentRef { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.name + .cmp(&other.name) + .then_with(|| self.range.start.cmp(&other.range.start)) + } +} + +impl fmt::Display for IdentRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}@{:?}", self.name, self.range) + } +} + +impl Serialize for IdentRef { + fn serialize(&self, serializer: S) -> Result { + let s = self.to_string(); + serializer.serialize_str(&s) + } +} + +impl<'de> Deserialize<'de> for IdentRef { + fn deserialize>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + let (name, range) = { + let mut parts = s.split('@'); + let name = parts.next().ok_or_else(|| { + serde::de::Error::custom("expected name@range, but found empty string") + })?; + let range = parts.next().ok_or_else(|| { + serde::de::Error::custom("expected name@range, but found no range") + })?; + // let range = range + // .parse() + // .map_err(|e| serde::de::Error::custom(format!("failed to parse range: + // {}", e)))?; + let st_ed = range + .split("..") + .map(|s| { + s.parse().map_err(|e| { + serde::de::Error::custom(format!("failed to parse range: {}", e)) + }) + }) + .collect::, _>>()?; + if st_ed.len() != 2 { + return Err(serde::de::Error::custom("expected range to have 2 parts")); + } + (name, st_ed[0]..st_ed[1]) + }; + Ok(IdentRef { + name: name.to_string(), + range, + }) + } +} diff --git a/crates/tinymist-query/src/analysis/module.rs b/crates/tinymist-query/src/syntax/module.rs similarity index 58% rename from crates/tinymist-query/src/analysis/module.rs rename to crates/tinymist-query/src/syntax/module.rs index 38be2b60..dfd571d9 100644 --- a/crates/tinymist-query/src/analysis/module.rs +++ b/crates/tinymist-query/src/syntax/module.rs @@ -1,8 +1,11 @@ -use std::{collections::HashMap, sync::Once}; +use std::{collections::HashMap, path::Path, sync::Once}; +use typst::syntax::VirtualPath; use typst_ts_core::{typst::prelude::EcoVec, TypstFileId}; -use super::{find_imports, AnalysisContext}; +use crate::prelude::AnalysisContext; + +use super::find_imports; pub struct ModuleDependency { pub dependencies: EcoVec, @@ -51,3 +54,35 @@ pub fn construct_module_dependencies( dependencies } + +pub fn scan_workspace_files(root: &Path) -> Vec { + let mut res = vec![]; + for path in walkdir::WalkDir::new(root).follow_links(false).into_iter() { + let Ok(de) = path else { + continue; + }; + if !de.file_type().is_file() { + continue; + } + if !de + .path() + .extension() + .is_some_and(|e| e == "typ" || e == "typc") + { + continue; + } + + let path = de.path(); + let relative_path = match path.strip_prefix(root) { + Ok(p) => p, + Err(err) => { + log::warn!("failed to strip prefix, path: {path:?}, root: {root:?}: {err}"); + continue; + } + }; + + res.push(TypstFileId::new(None, VirtualPath::new(relative_path))); + } + + res +} diff --git a/crates/tinymist-query/src/tests.rs b/crates/tinymist-query/src/tests.rs index 1dcb42b0..bb732e94 100644 --- a/crates/tinymist-query/src/tests.rs +++ b/crates/tinymist-query/src/tests.rs @@ -48,7 +48,7 @@ pub fn snapshot_testing2(name: &str, f: &impl Fn(&mut AnalysisContext, PathBuf)) ) }) .collect::>(); - let mut ctx = AnalysisContext::new(w); + let mut ctx = AnalysisContext::new(w, PositionEncoding::Utf16); ctx.test_files(|| paths); f(&mut ctx, p); }); diff --git a/crates/tinymist/src/actor/typst.rs b/crates/tinymist/src/actor/typst.rs index 621744dc..57656f4c 100644 --- a/crates/tinymist/src/actor/typst.rs +++ b/crates/tinymist/src/actor/typst.rs @@ -11,7 +11,8 @@ use once_cell::sync::OnceCell; use parking_lot::Mutex; use tinymist_query::{ analysis::AnalysisContext, CompilerQueryRequest, CompilerQueryResponse, DiagnosticsMap, - FoldRequestFeature, OnExportRequest, OnSaveExportRequest, PositionEncoding, VersionedDocument, + FoldRequestFeature, OnExportRequest, OnSaveExportRequest, PositionEncoding, SyntaxRequest, + VersionedDocument, }; use tokio::sync::{broadcast, mpsc, oneshot, watch}; use typst::{ @@ -205,8 +206,7 @@ macro_rules! query_world { macro_rules! query_world2 { ($self:ident, $method:ident, $req:expr) => {{ - let enc = $self.position_encoding; - let res = $self.steal_world2(move |w| $req.request(w, enc)); + let res = $self.steal_world2(move |w| $req.request(w)); res.map(CompilerQueryResponse::$method) }}; } @@ -751,9 +751,10 @@ impl CompileActor { &self, f: impl FnOnce(&mut AnalysisContext) -> T + Send + Sync + 'static, ) -> anyhow::Result { + let enc = self.position_encoding; let fut = self.steal(move |compiler| { // todo: record analysis - f(&mut AnalysisContext::new(compiler.compiler.world())) + f(&mut AnalysisContext::new(compiler.compiler.world(), enc)) }); Ok(fut?)