diff --git a/Cargo.lock b/Cargo.lock index 714d1fcc..8b8437ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3366,27 +3366,43 @@ dependencies = [ "env_logger", "futures", "itertools 0.12.1", - "lazy_static", "log", "once_cell", "parking_lot", - "percent-encoding", - "regex", "serde", "serde_json", - "strum", - "thiserror", + "tinymist-query", "tokio", "tower-lsp", "typst", "typst-assets", - "typst-ide", "typst-pdf", "typst-preview", "typst-ts-compiler", "typst-ts-core", ] +[[package]] +name = "tinymist-query" +version = "0.1.0" +dependencies = [ + "anyhow", + "comemo", + "itertools 0.12.1", + "lazy_static", + "log", + "parking_lot", + "regex", + "serde", + "serde_json", + "strum", + "tower-lsp", + "typst", + "typst-ide", + "typst-ts-compiler", + "typst-ts-core", +] + [[package]] name = "tinystr" version = "0.7.5" diff --git a/crates/tinymist-query/Cargo.toml b/crates/tinymist-query/Cargo.toml new file mode 100644 index 00000000..28b995e7 --- /dev/null +++ b/crates/tinymist-query/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "tinymist-query" +description = "Language queries for tinymist." +categories = ["compilers", "command-line-utilities"] +keywords = ["api", "language", "typst"] +authors.workspace = true +version.workspace = true +license.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true + +[dependencies] + +anyhow.workspace = true +comemo.workspace = true +regex.workspace = true +itertools.workspace = true +lazy_static.workspace = true +strum.workspace = true +log.workspace = true +serde.workspace = true +serde_json.workspace = true +parking_lot.workspace = true + +typst.workspace = true +typst-ide.workspace = true + +typst-ts-core = { version = "0.4.2-rc6", default-features = false, features = [ + "flat-vector", + "vector-bbox", +] } +typst-ts-compiler.workspace = true + +tower-lsp.workspace = true + +# [lints] +# workspace = true diff --git a/crates/tinymist/src/analysis/analyze.rs b/crates/tinymist-query/src/analysis/analyze.rs similarity index 100% rename from crates/tinymist/src/analysis/analyze.rs rename to crates/tinymist-query/src/analysis/analyze.rs diff --git a/crates/tinymist/src/analysis/mod.rs b/crates/tinymist-query/src/analysis/mod.rs similarity index 100% rename from crates/tinymist/src/analysis/mod.rs rename to crates/tinymist-query/src/analysis/mod.rs diff --git a/crates/tinymist-query/src/completion.rs b/crates/tinymist-query/src/completion.rs new file mode 100644 index 00000000..49a2724f --- /dev/null +++ b/crates/tinymist-query/src/completion.rs @@ -0,0 +1,27 @@ +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct CompletionRequest { + pub path: PathBuf, + pub position: LspPosition, + pub position_encoding: PositionEncoding, + pub explicit: bool, +} + +pub fn completion( + world: &TypstSystemWorld, + doc: Option>, + req: CompletionRequest, +) -> Option { + let source = get_suitable_source_in_workspace(world, &req.path).ok()?; + let typst_offset = + lsp_to_typst::position_to_offset(req.position, req.position_encoding, &source); + + let (typst_start_offset, completions) = + typst_ide::autocomplete(world, doc.as_deref(), &source, typst_offset, req.explicit)?; + + let lsp_start_position = + typst_to_lsp::offset_to_position(typst_start_offset, req.position_encoding, &source); + let replace_range = LspRawRange::new(lsp_start_position, req.position); + Some(typst_to_lsp::completions(&completions, replace_range).into()) +} diff --git a/crates/tinymist-query/src/diagnostics.rs b/crates/tinymist-query/src/diagnostics.rs new file mode 100644 index 00000000..ed4e16a6 --- /dev/null +++ b/crates/tinymist-query/src/diagnostics.rs @@ -0,0 +1,143 @@ +use crate::prelude::*; + +pub type DiagnosticsMap = HashMap>; + +pub fn convert_diagnostics<'a>( + project: &TypstSystemWorld, + errors: impl IntoIterator, + position_encoding: PositionEncoding, +) -> DiagnosticsMap { + errors + .into_iter() + .flat_map(|error| { + convert_diagnostic(project, error, position_encoding) + .map_err(move |conversion_err| { + error!("could not convert Typst error to diagnostic: {conversion_err:?} error to convert: {error:?}"); + }) + }) + .collect::>() + .into_iter() + .into_group_map() +} + +fn convert_diagnostic( + project: &TypstSystemWorld, + typst_diagnostic: &TypstDiagnostic, + position_encoding: PositionEncoding, +) -> anyhow::Result<(Url, LspDiagnostic)> { + let uri; + let lsp_range; + if let Some((id, span)) = diagnostic_span_id(typst_diagnostic) { + uri = Url::from_file_path(project.path_for_id(id)?).unwrap(); + let source = project.source(id)?; + lsp_range = diagnostic_range(&source, span, position_encoding).raw_range; + } else { + uri = Url::from_file_path(project.root.clone()).unwrap(); + lsp_range = LspRawRange::default(); + }; + + let lsp_severity = diagnostic_severity(typst_diagnostic.severity); + + let typst_message = &typst_diagnostic.message; + let typst_hints = &typst_diagnostic.hints; + let lsp_message = format!("{typst_message}{}", diagnostic_hints(typst_hints)); + + let tracepoints = diagnostic_related_information(project, typst_diagnostic, position_encoding)?; + + let diagnostic = LspDiagnostic { + range: lsp_range, + severity: Some(lsp_severity), + message: lsp_message, + source: Some("typst".to_owned()), + related_information: Some(tracepoints), + ..Default::default() + }; + + Ok((uri, diagnostic)) +} + +fn tracepoint_to_relatedinformation( + project: &TypstSystemWorld, + tracepoint: &Spanned, + position_encoding: PositionEncoding, +) -> anyhow::Result> { + if let Some(id) = tracepoint.span.id() { + let uri = Url::from_file_path(project.path_for_id(id)?).unwrap(); + let source = project.source(id)?; + + if let Some(typst_range) = source.range(tracepoint.span) { + let lsp_range = typst_to_lsp::range(typst_range, &source, position_encoding); + + return Ok(Some(DiagnosticRelatedInformation { + location: LspLocation { + uri, + range: lsp_range.raw_range, + }, + message: tracepoint.v.to_string(), + })); + } + } + + Ok(None) +} + +fn diagnostic_related_information( + project: &TypstSystemWorld, + typst_diagnostic: &TypstDiagnostic, + position_encoding: PositionEncoding, +) -> anyhow::Result> { + let mut tracepoints = vec![]; + + for tracepoint in &typst_diagnostic.trace { + if let Some(info) = + tracepoint_to_relatedinformation(project, tracepoint, position_encoding)? + { + tracepoints.push(info); + } + } + + Ok(tracepoints) +} + +fn diagnostic_span_id(typst_diagnostic: &TypstDiagnostic) -> Option<(FileId, TypstSpan)> { + iter::once(typst_diagnostic.span) + .chain(typst_diagnostic.trace.iter().map(|trace| trace.span)) + .find_map(|span| Some((span.id()?, span))) +} + +fn diagnostic_range( + source: &Source, + typst_span: TypstSpan, + position_encoding: PositionEncoding, +) -> LspRange { + // Due to #241 and maybe typst/typst#2035, we sometimes fail to find the span. + // In that case, we use a default span as a better alternative to + // panicking. + // + // This may have been fixed after Typst 0.7.0, but it's still nice to avoid + // panics in case something similar reappears. + match source.find(typst_span) { + Some(node) => { + let typst_range = node.range(); + typst_to_lsp::range(typst_range, source, position_encoding) + } + None => LspRange::new( + LspRawRange::new(LspPosition::new(0, 0), LspPosition::new(0, 0)), + position_encoding, + ), + } +} + +fn diagnostic_severity(typst_severity: TypstSeverity) -> LspSeverity { + match typst_severity { + TypstSeverity::Error => LspSeverity::ERROR, + TypstSeverity::Warning => LspSeverity::WARNING, + } +} + +fn diagnostic_hints(typst_hints: &[EcoString]) -> Format + '_> { + iter::repeat(EcoString::from("\n\nHint: ")) + .take(typst_hints.len()) + .interleave(typst_hints.iter().cloned()) + .format("") +} diff --git a/crates/tinymist-query/src/document_symbol.rs b/crates/tinymist-query/src/document_symbol.rs new file mode 100644 index 00000000..ee8d541f --- /dev/null +++ b/crates/tinymist-query/src/document_symbol.rs @@ -0,0 +1,160 @@ +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct DocumentSymbolRequest { + pub path: PathBuf, + pub position_encoding: PositionEncoding, +} + +pub fn document_symbol( + world: &TypstSystemWorld, + req: DocumentSymbolRequest, +) -> Option { + let source = get_suitable_source_in_workspace(world, &req.path).ok()?; + + let uri = Url::from_file_path(req.path).unwrap(); + let symbols = get_document_symbols(source, uri, req.position_encoding); + + symbols.map(DocumentSymbolResponse::Flat) +} + +#[comemo::memoize] +pub(crate) fn get_document_symbols( + source: Source, + uri: Url, + position_encoding: PositionEncoding, +) -> Option> { + struct DocumentSymbolWorker { + symbols: Vec, + } + + impl DocumentSymbolWorker { + /// Get all symbols for a node recursively. + pub fn get_symbols<'a>( + &mut self, + node: LinkedNode<'a>, + source: &'a Source, + uri: &'a Url, + position_encoding: PositionEncoding, + ) -> anyhow::Result<()> { + let own_symbol = get_ident(&node, source, uri, position_encoding)?; + + for child in node.children() { + self.get_symbols(child, source, uri, position_encoding)?; + } + + if let Some(symbol) = own_symbol { + self.symbols.push(symbol); + } + + Ok(()) + } + } + + /// Get symbol for a leaf node of a valid type, or `None` if the node is an + /// invalid type. + #[allow(deprecated)] + fn get_ident( + node: &LinkedNode, + source: &Source, + uri: &Url, + position_encoding: PositionEncoding, + ) -> anyhow::Result> { + match node.kind() { + SyntaxKind::Label => { + let ast_node = node + .cast::() + .ok_or_else(|| anyhow!("cast to ast node failed: {:?}", node))?; + let name = ast_node.get().to_string(); + let symbol = SymbolInformation { + name, + kind: SymbolKind::CONSTANT, + tags: None, + deprecated: None, // do not use, deprecated, use `tags` instead + location: LspLocation { + uri: uri.clone(), + range: typst_to_lsp::range(node.range(), source, position_encoding) + .raw_range, + }, + container_name: None, + }; + Ok(Some(symbol)) + } + SyntaxKind::Ident => { + let ast_node = node + .cast::() + .ok_or_else(|| anyhow!("cast to ast node failed: {:?}", node))?; + let name = ast_node.get().to_string(); + let Some(parent) = node.parent() else { + return Ok(None); + }; + let kind = match parent.kind() { + // for variable definitions, the Let binding holds an Ident + SyntaxKind::LetBinding => SymbolKind::VARIABLE, + // for function definitions, the Let binding holds a Closure which holds the + // Ident + SyntaxKind::Closure => { + let Some(grand_parent) = parent.parent() else { + return Ok(None); + }; + match grand_parent.kind() { + SyntaxKind::LetBinding => SymbolKind::FUNCTION, + _ => return Ok(None), + } + } + _ => return Ok(None), + }; + let symbol = SymbolInformation { + name, + kind, + tags: None, + deprecated: None, // do not use, deprecated, use `tags` instead + location: LspLocation { + uri: uri.clone(), + range: typst_to_lsp::range(node.range(), source, position_encoding) + .raw_range, + }, + container_name: None, + }; + Ok(Some(symbol)) + } + SyntaxKind::Markup => { + let name = node.get().to_owned().into_text().to_string(); + if name.is_empty() { + return Ok(None); + } + let Some(parent) = node.parent() else { + return Ok(None); + }; + let kind = match parent.kind() { + SyntaxKind::Heading => SymbolKind::NAMESPACE, + _ => return Ok(None), + }; + let symbol = SymbolInformation { + name, + kind, + tags: None, + deprecated: None, // do not use, deprecated, use `tags` instead + location: LspLocation { + uri: uri.clone(), + range: typst_to_lsp::range(node.range(), source, position_encoding) + .raw_range, + }, + container_name: None, + }; + Ok(Some(symbol)) + } + _ => Ok(None), + } + } + + let root = LinkedNode::new(source.root()); + + let mut worker = DocumentSymbolWorker { symbols: vec![] }; + + let res = worker + .get_symbols(root, &source, &uri, position_encoding) + .ok(); + + res.map(|_| worker.symbols) +} diff --git a/crates/tinymist-query/src/hover.rs b/crates/tinymist-query/src/hover.rs new file mode 100644 index 00000000..21948a01 --- /dev/null +++ b/crates/tinymist-query/src/hover.rs @@ -0,0 +1,28 @@ +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct HoverRequest { + pub path: PathBuf, + pub position: LspPosition, + pub position_encoding: PositionEncoding, +} + +pub fn hover( + world: &TypstSystemWorld, + doc: Option>, + req: HoverRequest, +) -> Option { + let source = get_suitable_source_in_workspace(world, &req.path).ok()?; + let typst_offset = + lsp_to_typst::position_to_offset(req.position, req.position_encoding, &source); + + let typst_tooltip = typst_ide::tooltip(world, doc.as_deref(), &source, typst_offset)?; + + let ast_node = LinkedNode::new(source.root()).leaf_at(typst_offset)?; + let range = typst_to_lsp::range(ast_node.range(), &source, req.position_encoding); + + Some(Hover { + contents: typst_to_lsp::tooltip(&typst_tooltip), + range: Some(range.raw_range), + }) +} diff --git a/crates/tinymist-query/src/lib.rs b/crates/tinymist-query/src/lib.rs new file mode 100644 index 00000000..c760d71a --- /dev/null +++ b/crates/tinymist-query/src/lib.rs @@ -0,0 +1,27 @@ +pub mod analysis; + +pub(crate) mod diagnostics; +pub use diagnostics::*; +pub(crate) mod signature_help; +pub use signature_help::*; +pub(crate) mod document_symbol; +pub use document_symbol::*; +pub(crate) mod symbol; +pub use symbol::*; +pub(crate) mod semantic_tokens; +pub use semantic_tokens::*; +pub(crate) mod semantic_tokens_full; +pub use semantic_tokens_full::*; +pub(crate) mod semantic_tokens_delta; +pub use semantic_tokens_delta::*; +pub(crate) mod hover; +pub use hover::*; +pub(crate) mod completion; +pub use completion::*; +pub(crate) mod selection_range; +pub use selection_range::*; + +pub mod lsp_typst_boundary; +pub use lsp_typst_boundary::*; + +mod prelude; diff --git a/crates/tinymist/src/lsp_typst_boundary.rs b/crates/tinymist-query/src/lsp_typst_boundary.rs similarity index 91% rename from crates/tinymist/src/lsp_typst_boundary.rs rename to crates/tinymist-query/src/lsp_typst_boundary.rs index e4249f7e..a274d9a0 100644 --- a/crates/tinymist/src/lsp_typst_boundary.rs +++ b/crates/tinymist-query/src/lsp_typst_boundary.rs @@ -7,7 +7,7 @@ pub type LspPosition = lsp_types::Position; /// The interpretation of an `LspCharacterOffset` depends on the /// `LspPositionEncoding` pub type LspCharacterOffset = u32; -pub type LspPositionEncoding = crate::config::PositionEncoding; +pub type LspPositionEncoding = PositionEncoding; /// Byte offset (i.e. UTF-8 bytes) in Typst files, either from the start of the /// line or the file pub type TypstOffset = usize; @@ -30,6 +30,33 @@ pub type TypstSeverity = typst::diag::Severity; pub type LspParamInfo = lsp_types::ParameterInformation; pub type TypstParamInfo = typst::foundations::ParamInfo; +/// What counts as "1 character" for string indexing. We should always prefer +/// UTF-8, but support UTF-16 as long as it is standard. For more background on +/// encodings and LSP, try ["The bottom emoji breaks rust-analyzer"](https://fasterthanli.me/articles/the-bottom-emoji-breaks-rust-analyzer), +/// a well-written article on the topic. +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Default)] +pub enum PositionEncoding { + /// "1 character" means "1 UTF-16 code unit" + /// + /// This is the only required encoding for LSPs to support, but it's not a + /// natural one (unless you're working in JS). Prefer UTF-8, and refer + /// to the article linked in the `PositionEncoding` docs for more + /// background. + #[default] + Utf16, + /// "1 character" means "1 byte" + Utf8, +} + +impl From for lsp_types::PositionEncodingKind { + fn from(position_encoding: PositionEncoding) -> Self { + match position_encoding { + PositionEncoding::Utf16 => Self::UTF16, + PositionEncoding::Utf8 => Self::UTF8, + } + } +} + /// An LSP range with its associated encoding. pub struct LspRange { pub raw_range: LspRawRange, @@ -289,8 +316,7 @@ pub mod typst_to_lsp { mod test { use typst::syntax::Source; - use crate::config::PositionEncoding; - use crate::lsp_typst_boundary::lsp_to_typst; + use crate::{lsp_to_typst, PositionEncoding}; use super::*; diff --git a/crates/tinymist-query/src/prelude.rs b/crates/tinymist-query/src/prelude.rs new file mode 100644 index 00000000..76afa628 --- /dev/null +++ b/crates/tinymist-query/src/prelude.rs @@ -0,0 +1,40 @@ +pub use std::{ + collections::HashMap, + iter, + path::{Path, PathBuf}, + sync::Arc, +}; + +pub use anyhow::anyhow; +pub use itertools::{Format, Itertools}; +pub use log::{error, trace}; +pub use tower_lsp::lsp_types::{ + CompletionResponse, DiagnosticRelatedInformation, DocumentSymbolResponse, Documentation, Hover, + Location as LspLocation, MarkupContent, MarkupKind, Position as LspPosition, SelectionRange, + SemanticTokens, SemanticTokensDelta, SemanticTokensFullDeltaResult, SemanticTokensResult, + SignatureHelp, SignatureInformation, SymbolInformation, SymbolKind, Url, +}; +pub use typst::diag::{EcoString, FileError, FileResult, Tracepoint}; +pub use typst::foundations::{Func, ParamInfo, Value}; +pub use typst::syntax::{ + ast::{self, AstNode}, + FileId, LinkedNode, Source, Spanned, SyntaxKind, VirtualPath, +}; +pub use typst::World; +use typst_ts_compiler::service::WorkspaceProvider; +pub use typst_ts_compiler::TypstSystemWorld; +pub use typst_ts_core::{TypstDocument, TypstFileId}; + +pub use crate::analysis::analyze::analyze_expr; +pub use crate::lsp_typst_boundary::{ + lsp_to_typst, typst_to_lsp, LspDiagnostic, LspRange, LspRawRange, LspSeverity, + PositionEncoding, TypstDiagnostic, TypstSeverity, TypstSpan, +}; + +pub fn get_suitable_source_in_workspace(w: &TypstSystemWorld, p: &Path) -> FileResult { + // todo: source in packages + let relative_path = p + .strip_prefix(&w.workspace_root()) + .map_err(|_| FileError::NotFound(p.to_owned()))?; + w.source(TypstFileId::new(None, VirtualPath::new(relative_path))) +} diff --git a/crates/tinymist-query/src/selection_range.rs b/crates/tinymist-query/src/selection_range.rs new file mode 100644 index 00000000..94b17da3 --- /dev/null +++ b/crates/tinymist-query/src/selection_range.rs @@ -0,0 +1,40 @@ +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct SelectionRangeRequest { + pub path: PathBuf, + pub positions: Vec, + pub position_encoding: PositionEncoding, +} + +pub fn selection_range( + world: &TypstSystemWorld, + req: SelectionRangeRequest, +) -> Option> { + let source = get_suitable_source_in_workspace(world, &req.path).ok()?; + + let mut ranges = Vec::new(); + for position in req.positions { + let typst_offset = + lsp_to_typst::position_to_offset(position, req.position_encoding, &source); + let tree = LinkedNode::new(source.root()); + let leaf = tree.leaf_at(typst_offset)?; + ranges.push(range_for_node(&source, req.position_encoding, &leaf)); + } + + Some(ranges) +} + +fn range_for_node( + source: &Source, + position_encoding: PositionEncoding, + node: &LinkedNode, +) -> SelectionRange { + let range = typst_to_lsp::range(node.range(), source, position_encoding); + SelectionRange { + range: range.raw_range, + parent: node + .parent() + .map(|node| Box::new(range_for_node(source, position_encoding, node))), + } +} diff --git a/crates/tinymist/src/semantic_tokens/delta.rs b/crates/tinymist-query/src/semantic_tokens/delta.rs similarity index 97% rename from crates/tinymist/src/semantic_tokens/delta.rs rename to crates/tinymist-query/src/semantic_tokens/delta.rs index 2eaf978a..1883efd4 100644 --- a/crates/tinymist/src/semantic_tokens/delta.rs +++ b/crates/tinymist-query/src/semantic_tokens/delta.rs @@ -7,12 +7,12 @@ struct CachedTokens { } #[derive(Default, Debug)] -pub struct Cache { +pub struct CacheInner { last_sent: Option, next_id: u64, } -impl Cache { +impl CacheInner { pub fn try_take_result(&mut self, id: &str) -> Option> { let id = id.parse::().ok()?; match self.last_sent.take() { diff --git a/crates/tinymist/src/semantic_tokens/mod.rs b/crates/tinymist-query/src/semantic_tokens/mod.rs similarity index 94% rename from crates/tinymist/src/semantic_tokens/mod.rs rename to crates/tinymist-query/src/semantic_tokens/mod.rs index 140357b7..a6368a8a 100644 --- a/crates/tinymist/src/semantic_tokens/mod.rs +++ b/crates/tinymist-query/src/semantic_tokens/mod.rs @@ -1,4 +1,5 @@ use itertools::Itertools; +use parking_lot::RwLock; use strum::IntoEnumIterator; use tower_lsp::lsp_types::{ Registration, SemanticToken, SemanticTokensEdit, SemanticTokensFullOptions, @@ -7,15 +8,14 @@ use tower_lsp::lsp_types::{ use typst::diag::EcoString; use typst::syntax::{ast, LinkedNode, Source, SyntaxKind}; -use crate::actor::typst::CompileCluster; -use crate::config::PositionEncoding; +use crate::PositionEncoding; use self::delta::token_delta; use self::modifier_set::ModifierSet; use self::token_encode::encode_tokens; use self::typst_tokens::{Modifier, TokenType}; -pub use self::delta::Cache as SemanticTokenCache; +pub use self::delta::CacheInner as TokenCacheInner; mod delta; mod modifier_set; @@ -58,7 +58,10 @@ pub fn get_semantic_tokens_options() -> SemanticTokensOptions { } } -impl CompileCluster { +#[derive(Default)] +pub struct SemanticTokenCache(RwLock); + +impl SemanticTokenCache { pub fn get_semantic_tokens_full( &self, source: &Source, @@ -70,10 +73,7 @@ impl CompileCluster { let encoded_tokens = encode_tokens(tokens, source, encoding); let output_tokens = encoded_tokens.map(|(token, _)| token).collect_vec(); - let result_id = self - .semantic_tokens_delta_cache - .write() - .cache_result(output_tokens.clone()); + let result_id = self.0.write().cache_result(output_tokens.clone()); (output_tokens, result_id) } @@ -84,10 +84,7 @@ impl CompileCluster { result_id: &str, encoding: PositionEncoding, ) -> (Result, Vec>, String) { - let cached = self - .semantic_tokens_delta_cache - .write() - .try_take_result(result_id); + let cached = self.0.write().try_take_result(result_id); // this call will overwrite the cache, so need to read from cache first let (tokens, result_id) = self.get_semantic_tokens_full(source, encoding); diff --git a/crates/tinymist/src/semantic_tokens/modifier_set.rs b/crates/tinymist-query/src/semantic_tokens/modifier_set.rs similarity index 100% rename from crates/tinymist/src/semantic_tokens/modifier_set.rs rename to crates/tinymist-query/src/semantic_tokens/modifier_set.rs diff --git a/crates/tinymist-query/src/semantic_tokens/token_encode.rs b/crates/tinymist-query/src/semantic_tokens/token_encode.rs new file mode 100644 index 00000000..7a8de69e --- /dev/null +++ b/crates/tinymist-query/src/semantic_tokens/token_encode.rs @@ -0,0 +1,86 @@ +use tower_lsp::lsp_types::{Position, SemanticToken}; +use typst::diag::EcoString; +use typst::syntax::Source; + +use crate::typst_to_lsp; +use crate::PositionEncoding; + +use super::Token; + +pub(super) fn encode_tokens<'a>( + tokens: impl Iterator + 'a, + source: &'a Source, + encoding: PositionEncoding, +) -> impl Iterator + 'a { + tokens.scan(Position::new(0, 0), move |last_position, token| { + let (encoded_token, source_code, position) = + encode_token(token, last_position, source, encoding); + *last_position = position; + Some((encoded_token, source_code)) + }) +} + +fn encode_token( + token: Token, + last_position: &Position, + source: &Source, + encoding: PositionEncoding, +) -> (SemanticToken, EcoString, Position) { + let position = typst_to_lsp::offset_to_position(token.offset, encoding, source); + let delta = last_position.delta(&position); + + let length = token.source.as_str().encoded_len(encoding); + + let lsp_token = SemanticToken { + delta_line: delta.delta_line, + delta_start: delta.delta_start, + length: length as u32, + token_type: token.token_type as u32, + token_modifiers_bitset: token.modifiers.bitset(), + }; + + (lsp_token, token.source, position) +} + +pub trait StrExt { + fn encoded_len(&self, encoding: PositionEncoding) -> usize; +} + +impl StrExt for str { + fn encoded_len(&self, encoding: PositionEncoding) -> usize { + match encoding { + PositionEncoding::Utf8 => self.len(), + PositionEncoding::Utf16 => self.chars().map(char::len_utf16).sum(), + } + } +} + +pub trait PositionExt { + fn delta(&self, to: &Self) -> PositionDelta; +} + +impl PositionExt for Position { + /// Calculates the delta from `self` to `to`. This is in the `SemanticToken` + /// sense, so the delta's `character` is relative to `self`'s + /// `character` iff `self` and `to` are on the same line. Otherwise, + /// it's relative to the start of the line `to` is on. + fn delta(&self, to: &Self) -> PositionDelta { + let line_delta = to.line - self.line; + let char_delta = if line_delta == 0 { + to.character - self.character + } else { + to.character + }; + + PositionDelta { + delta_line: line_delta, + delta_start: char_delta, + } + } +} + +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Copy, Clone, Default)] +pub struct PositionDelta { + pub delta_line: u32, + pub delta_start: u32, +} diff --git a/crates/tinymist/src/semantic_tokens/typst_tokens.rs b/crates/tinymist-query/src/semantic_tokens/typst_tokens.rs similarity index 100% rename from crates/tinymist/src/semantic_tokens/typst_tokens.rs rename to crates/tinymist-query/src/semantic_tokens/typst_tokens.rs diff --git a/crates/tinymist-query/src/semantic_tokens_delta.rs b/crates/tinymist-query/src/semantic_tokens_delta.rs new file mode 100644 index 00000000..6f6c3c7e --- /dev/null +++ b/crates/tinymist-query/src/semantic_tokens_delta.rs @@ -0,0 +1,37 @@ +use crate::{prelude::*, SemanticTokenCache}; + +#[derive(Debug, Clone)] +pub struct SemanticTokensDeltaRequest { + pub path: PathBuf, + pub previous_result_id: String, + pub position_encoding: PositionEncoding, +} + +pub fn semantic_tokens_delta( + cache: &SemanticTokenCache, + source: Source, + req: SemanticTokensDeltaRequest, +) -> Option { + let (tokens, result_id) = cache.try_semantic_tokens_delta_from_result_id( + &source, + &req.previous_result_id, + req.position_encoding, + ); + + match tokens { + Ok(edits) => Some( + SemanticTokensDelta { + result_id: Some(result_id), + edits, + } + .into(), + ), + Err(tokens) => Some( + SemanticTokens { + result_id: Some(result_id), + data: tokens, + } + .into(), + ), + } +} diff --git a/crates/tinymist-query/src/semantic_tokens_full.rs b/crates/tinymist-query/src/semantic_tokens_full.rs new file mode 100644 index 00000000..8dcf9998 --- /dev/null +++ b/crates/tinymist-query/src/semantic_tokens_full.rs @@ -0,0 +1,23 @@ +use crate::{prelude::*, SemanticTokenCache}; + +#[derive(Debug, Clone)] +pub struct SemanticTokensFullRequest { + pub path: PathBuf, + pub position_encoding: PositionEncoding, +} + +pub fn semantic_tokens_full( + cache: &SemanticTokenCache, + source: Source, + req: SemanticTokensFullRequest, +) -> Option { + let (tokens, result_id) = cache.get_semantic_tokens_full(&source, req.position_encoding); + + Some( + SemanticTokens { + result_id: Some(result_id), + data: tokens, + } + .into(), + ) +} diff --git a/crates/tinymist-query/src/signature_help.rs b/crates/tinymist-query/src/signature_help.rs new file mode 100644 index 00000000..e9c7eae1 --- /dev/null +++ b/crates/tinymist-query/src/signature_help.rs @@ -0,0 +1,165 @@ +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct SignatureHelpRequest { + pub path: PathBuf, + pub position: LspPosition, + pub position_encoding: PositionEncoding, +} + +pub fn signature_help( + world: &TypstSystemWorld, + SignatureHelpRequest { + path, + position, + position_encoding, + }: SignatureHelpRequest, +) -> Option { + let source = get_suitable_source_in_workspace(world, &path).ok()?; + let typst_offset = lsp_to_typst::position_to_offset(position, position_encoding, &source); + + let ast_node = LinkedNode::new(source.root()).leaf_at(typst_offset)?; + let (callee, callee_node, args) = surrounding_function_syntax(&ast_node)?; + + let mut ancestor = &ast_node; + while !ancestor.is::() { + ancestor = ancestor.parent()?; + } + + if !callee.hash() && !matches!(callee, ast::Expr::MathIdent(_)) { + return None; + } + + let values = analyze_expr(world, &callee_node); + + let function = values.into_iter().find_map(|v| match v { + Value::Func(f) => Some(f), + _ => None, + })?; + trace!("got function {function:?}"); + + let param_index = param_index_at_leaf(&ast_node, &function, args); + + let label = format!( + "{}({}){}", + function.name().unwrap_or(""), + match function.params() { + Some(params) => params + .iter() + .map(typst_to_lsp::param_info_to_label) + .join(", "), + None => "".to_owned(), + }, + match function.returns() { + Some(returns) => format!("-> {}", typst_to_lsp::cast_info_to_label(returns)), + None => "".to_owned(), + } + ); + let params = function + .params() + .unwrap_or_default() + .iter() + .map(typst_to_lsp::param_info) + .collect(); + trace!("got signature info {label} {params:?}"); + + let documentation = function.docs().map(markdown_docs); + + let active_parameter = param_index.map(|i| i as u32); + + Some(SignatureHelp { + signatures: vec![SignatureInformation { + label, + documentation, + parameters: Some(params), + active_parameter, + }], + active_signature: Some(0), + active_parameter: None, + }) +} + +fn surrounding_function_syntax<'b>( + leaf: &'b LinkedNode, +) -> Option<(ast::Expr<'b>, LinkedNode<'b>, ast::Args<'b>)> { + let parent = leaf.parent()?; + let parent = match parent.kind() { + SyntaxKind::Named => parent.parent()?, + _ => parent, + }; + let args = parent.cast::()?; + let grand = parent.parent()?; + let expr = grand.cast::()?; + let callee = match expr { + ast::Expr::FuncCall(call) => call.callee(), + ast::Expr::Set(set) => set.target(), + _ => return None, + }; + Some((callee, grand.find(callee.span())?, args)) +} + +fn param_index_at_leaf(leaf: &LinkedNode, function: &Func, args: ast::Args) -> Option { + let deciding = deciding_syntax(leaf); + let params = function.params()?; + let param_index = find_param_index(&deciding, params, args)?; + trace!("got param index {param_index}"); + Some(param_index) +} + +/// Find the piece of syntax that decides what we're completing. +fn deciding_syntax<'b>(leaf: &'b LinkedNode) -> LinkedNode<'b> { + let mut deciding = leaf.clone(); + while !matches!( + deciding.kind(), + SyntaxKind::LeftParen | SyntaxKind::Comma | SyntaxKind::Colon + ) { + let Some(prev) = deciding.prev_leaf() else { + break; + }; + deciding = prev; + } + deciding +} + +fn find_param_index(deciding: &LinkedNode, params: &[ParamInfo], args: ast::Args) -> Option { + match deciding.kind() { + // After colon: "func(param:|)", "func(param: |)". + SyntaxKind::Colon => { + let prev = deciding.prev_leaf()?; + let param_ident = prev.cast::()?; + params + .iter() + .position(|param| param.name == param_ident.as_str()) + } + // Before: "func(|)", "func(hi|)", "func(12,|)". + SyntaxKind::Comma | SyntaxKind::LeftParen => { + let next = deciding.next_leaf(); + let following_param = next.as_ref().and_then(|next| next.cast::()); + match following_param { + Some(next) => params + .iter() + .position(|param| param.named && param.name.starts_with(next.as_str())), + None => { + let positional_args_so_far = args + .items() + .filter(|arg| matches!(arg, ast::Arg::Pos(_))) + .count(); + params + .iter() + .enumerate() + .filter(|(_, param)| param.positional) + .map(|(i, _)| i) + .nth(positional_args_so_far) + } + } + } + _ => None, + } +} + +fn markdown_docs(docs: &str) -> Documentation { + Documentation::MarkupContent(MarkupContent { + kind: MarkupKind::Markdown, + value: docs.to_owned(), + }) +} diff --git a/crates/tinymist-query/src/symbol.rs b/crates/tinymist-query/src/symbol.rs new file mode 100644 index 00000000..2fd1d63a --- /dev/null +++ b/crates/tinymist-query/src/symbol.rs @@ -0,0 +1,50 @@ +use typst_ts_compiler::NotifyApi; + +use crate::document_symbol::get_document_symbols; +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct SymbolRequest { + pub pattern: Option, + pub position_encoding: PositionEncoding, +} + +pub fn symbol( + world: &TypstSystemWorld, + SymbolRequest { + pattern, + position_encoding, + }: SymbolRequest, +) -> Option> { + // todo: expose source + + let mut symbols = vec![]; + + world.iter_dependencies(&mut |path, _| { + let Ok(source) = get_suitable_source_in_workspace(world, path) else { + return; + }; + let uri = Url::from_file_path(path).unwrap(); + let res = get_document_symbols(source, uri, position_encoding).and_then(|symbols| { + pattern + .as_ref() + .map(|pattern| filter_document_symbols(symbols, pattern)) + }); + + if let Some(mut res) = res { + symbols.append(&mut res) + } + }); + + Some(symbols) +} + +fn filter_document_symbols( + symbols: Vec, + query_string: &str, +) -> Vec { + symbols + .into_iter() + .filter(|e| e.name.contains(query_string)) + .collect() +} diff --git a/crates/tinymist/Cargo.toml b/crates/tinymist/Cargo.toml index 8470e8c4..38b6e698 100644 --- a/crates/tinymist/Cargo.toml +++ b/crates/tinymist/Cargo.toml @@ -20,26 +20,22 @@ doc = false [dependencies] +tinymist-query = { path = "../tinymist-query" } + once_cell.workspace = true anyhow.workspace = true comemo.workspace = true -thiserror.workspace = true tokio.workspace = true futures.workspace = true -regex.workspace = true itertools.workspace = true -lazy_static.workspace = true -strum.workspace = true async-trait.workspace = true env_logger.workspace = true log.workspace = true -percent-encoding.workspace = true serde.workspace = true serde_json.workspace = true parking_lot.workspace = true typst.workspace = true -typst-ide.workspace = true typst-pdf.workspace = true typst-assets = { workspace = true, features = ["fonts"] } diff --git a/crates/tinymist/src/actor/render.rs b/crates/tinymist/src/actor/render.rs index 2b1f1ed4..cb6168ed 100644 --- a/crates/tinymist/src/actor/render.rs +++ b/crates/tinymist/src/actor/render.rs @@ -11,7 +11,7 @@ use tokio::sync::{ }; use typst_ts_core::TypstDocument; -use crate::config::ExportPdfMode; +use crate::lsp::ExportPdfMode; #[derive(Debug, Clone)] pub enum RenderActorRequest { diff --git a/crates/tinymist/src/actor/typst.rs b/crates/tinymist/src/actor/typst.rs index 2f13b87c..1052fa31 100644 --- a/crates/tinymist/src/actor/typst.rs +++ b/crates/tinymist/src/actor/typst.rs @@ -1,30 +1,23 @@ use std::{ collections::HashMap, - iter, path::{Path, PathBuf}, sync::{Arc, Mutex as SyncMutex}, }; use anyhow::anyhow; use futures::future::join_all; -use itertools::{Format, Itertools}; use log::{error, trace, warn}; +use tinymist_query::{LspDiagnostic, LspRange, PositionEncoding, SemanticTokenCache}; use tokio::sync::{broadcast, mpsc, watch, Mutex, RwLock}; use tower_lsp::lsp_types::{ - CompletionResponse, DiagnosticRelatedInformation, DocumentSymbolResponse, Documentation, Hover, - Location as LspLocation, MarkupContent, MarkupKind, Position as LspPosition, SelectionRange, - SemanticTokens, SemanticTokensDelta, SemanticTokensFullDeltaResult, SemanticTokensResult, - SignatureHelp, SignatureInformation, SymbolInformation, SymbolKind, + CompletionResponse, DocumentSymbolResponse, Hover, SelectionRange, + SemanticTokensFullDeltaResult, SemanticTokensResult, SignatureHelp, SymbolInformation, TextDocumentContentChangeEvent, Url, }; -use typst::diag::{EcoString, FileError, FileResult, SourceDiagnostic, SourceResult, Tracepoint}; -use typst::foundations::{Func, ParamInfo, Value}; +use typst::diag::{FileResult, SourceDiagnostic, SourceResult}; use typst::layout::Position; use typst::model::Document; -use typst::syntax::{ - ast::{self, AstNode}, - FileId, LinkedNode, Source, Span, Spanned, SyntaxKind, VirtualPath, -}; +use typst::syntax::{Source, Span}; use typst::World; use typst_preview::CompilationHandleImpl; use typst_preview::{CompilationHandle, CompileStatus}; @@ -35,34 +28,25 @@ use typst_ts_compiler::service::{ CompileExporter, CompileMiddleware, Compiler, WorkspaceProvider, WorldExporter, }; use typst_ts_compiler::vfs::notify::{FileChangeSet, MemoryEvent}; -use typst_ts_compiler::{NotifyApi, Time, TypstSystemWorld}; +use typst_ts_compiler::{Time, TypstSystemWorld}; use typst_ts_core::{ config::CompileOpts, debug_loc::SourceSpanOffset, error::prelude::*, typst::prelude::EcoVec, - Bytes, DynExporter, Error, ImmutPath, TypstDocument, TypstFileId, + Bytes, DynExporter, Error, ImmutPath, TypstDocument, }; use crate::actor::render::PdfExportActor; use crate::actor::render::RenderActorRequest; -use crate::analysis::analyze::analyze_expr; -use crate::config::PositionEncoding; use crate::lsp::LspHost; -use crate::lsp_typst_boundary::{ - lsp_to_typst, typst_to_lsp, LspDiagnostic, LspRange, LspRawRange, LspSeverity, TypstDiagnostic, - TypstSeverity, TypstSpan, -}; -use crate::semantic_tokens::SemanticTokenCache; type CompileService = CompileActor, H>>; type CompileClient = TsCompileClient>; type DiagnosticsSender = mpsc::UnboundedSender<(String, DiagnosticsMap)>; - type DiagnosticsMap = HashMap>; -pub type Client = TypstClient; +type Client = TypstClient; pub fn create_cluster(host: LspHost, roots: Vec, opts: CompileOpts) -> CompileCluster { - // let (diag_tx, diag_rx) = mpsc::unbounded_channel(); let primary = create_server( @@ -74,7 +58,7 @@ pub fn create_cluster(host: LspHost, roots: Vec, opts: CompileOpts) -> CompileCluster { memory_changes: RwLock::new(HashMap::new()), primary, - semantic_tokens_delta_cache: Default::default(), + tokens_cache: Default::default(), actor: Some(CompileClusterActor { host, diag_rx, @@ -130,7 +114,7 @@ pub struct CompileClusterActor { pub struct CompileCluster { memory_changes: RwLock, MemoryFileMeta>>, primary: CompileNode, - pub semantic_tokens_delta_cache: Arc>, + pub tokens_cache: SemanticTokenCache, actor: Option, } @@ -304,71 +288,17 @@ pub struct OnSaveExportRequest { pub path: PathBuf, } -#[derive(Debug, Clone)] -pub struct HoverRequest { - pub path: PathBuf, - pub position: LspPosition, - pub position_encoding: PositionEncoding, -} - -#[derive(Debug, Clone)] -pub struct CompletionRequest { - pub path: PathBuf, - pub position: LspPosition, - pub position_encoding: PositionEncoding, - pub explicit: bool, -} - -#[derive(Debug, Clone)] -pub struct SignatureHelpRequest { - pub path: PathBuf, - pub position: LspPosition, - pub position_encoding: PositionEncoding, -} - -#[derive(Debug, Clone)] -pub struct DocumentSymbolRequest { - pub path: PathBuf, - pub position_encoding: PositionEncoding, -} - -#[derive(Debug, Clone)] -pub struct SymbolRequest { - pub pattern: Option, - pub position_encoding: PositionEncoding, -} - -#[derive(Debug, Clone)] -pub struct SelectionRangeRequest { - pub path: PathBuf, - pub positions: Vec, - pub position_encoding: PositionEncoding, -} - -#[derive(Debug, Clone)] -pub struct SemanticTokensFullRequest { - pub path: PathBuf, - pub position_encoding: PositionEncoding, -} - -#[derive(Debug, Clone)] -pub struct SemanticTokensDeltaRequest { - pub path: PathBuf, - pub previous_result_id: String, - pub position_encoding: PositionEncoding, -} - #[derive(Debug, Clone)] pub enum CompilerQueryRequest { OnSaveExport(OnSaveExportRequest), - Hover(HoverRequest), - Completion(CompletionRequest), - SignatureHelp(SignatureHelpRequest), - DocumentSymbol(DocumentSymbolRequest), - Symbol(SymbolRequest), - SemanticTokensFull(SemanticTokensFullRequest), - SemanticTokensDelta(SemanticTokensDeltaRequest), - SelectionRange(SelectionRangeRequest), + Hover(tinymist_query::HoverRequest), + Completion(tinymist_query::CompletionRequest), + SignatureHelp(tinymist_query::SignatureHelpRequest), + DocumentSymbol(tinymist_query::DocumentSymbolRequest), + Symbol(tinymist_query::SymbolRequest), + SemanticTokensFull(tinymist_query::SemanticTokensFullRequest), + SemanticTokensDelta(tinymist_query::SemanticTokensDeltaRequest), + SelectionRange(tinymist_query::SelectionRangeRequest), } #[derive(Debug, Clone)] @@ -384,98 +314,49 @@ pub enum CompilerQueryResponse { SelectionRange(Option>), } +macro_rules! query_state { + ($self:ident, $method:ident, $query:expr, $req:expr) => {{ + let doc = $self.handler.result.lock().unwrap().clone().ok(); + let res = $self.steal_world(|w| $query(w, doc, $req)).await; + res.map(CompilerQueryResponse::$method) + }}; +} + +macro_rules! query_world { + ($self:ident, $method:ident, $query:expr, $req:expr) => {{ + let res = $self.steal_world(|w| $query(w, $req)).await; + res.map(CompilerQueryResponse::$method) + }}; +} + +macro_rules! query_tokens_cache { + ($self:ident, $method:ident, $query:expr, $req:expr) => {{ + let path: ImmutPath = $req.path.clone().into(); + let vfs = $self.memory_changes.read().await; + let snapshot = vfs.get(&path).ok_or_else(|| anyhow!("file missing"))?; + let res = $query(&$self.tokens_cache, snapshot.content.clone(), $req); + Ok(CompilerQueryResponse::$method(res)) + }}; +} + impl CompileCluster { pub async fn query( &self, query: CompilerQueryRequest, ) -> anyhow::Result { + use tinymist_query::*; + use CompilerQueryRequest::*; + match query { - CompilerQueryRequest::SemanticTokensFull(SemanticTokensFullRequest { - path, - position_encoding, - }) => self - .semantic_tokens_full(path, position_encoding) - .await - .map(CompilerQueryResponse::SemanticTokensFull), - CompilerQueryRequest::SemanticTokensDelta(SemanticTokensDeltaRequest { - path, - previous_result_id, - position_encoding, - }) => self - .semantic_tokens_delta(path, previous_result_id, position_encoding) - .await - .map(CompilerQueryResponse::SemanticTokensDelta), + SemanticTokensFull(req) => { + query_tokens_cache!(self, SemanticTokensFull, semantic_tokens_full, req) + } + SemanticTokensDelta(req) => { + query_tokens_cache!(self, SemanticTokensDelta, semantic_tokens_delta, req) + } _ => self.primary.query(query).await, } } - - async fn semantic_tokens_full( - &self, - path: PathBuf, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - let path: ImmutPath = path.into(); - - let source = self - .memory_changes - .read() - .await - .get(&path) - .ok_or_else(|| anyhow!("file missing"))? - .content - .clone(); - - let (tokens, result_id) = self.get_semantic_tokens_full(&source, position_encoding); - - Ok(Some( - SemanticTokens { - result_id: Some(result_id), - data: tokens, - } - .into(), - )) - } - - async fn semantic_tokens_delta( - &self, - path: PathBuf, - previous_result_id: String, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - let path: ImmutPath = path.into(); - - let source = self - .memory_changes - .read() - .await - .get(&path) - .ok_or_else(|| anyhow!("file missing"))? - .content - .clone(); - - let (tokens, result_id) = self.try_semantic_tokens_delta_from_result_id( - &source, - &previous_result_id, - position_encoding, - ); - - Ok(match tokens { - Ok(edits) => Some( - SemanticTokensDelta { - result_id: Some(result_id), - edits, - } - .into(), - ), - Err(tokens) => Some( - SemanticTokens { - result_id: Some(result_id), - data: tokens, - } - .into(), - ), - }) - } } #[derive(Clone)] @@ -565,6 +446,43 @@ pub struct CompileServer { client: TypstClient, } +impl CompileServer { + pub fn new( + diag_group: String, + compiler_driver: CompileDriver, + cb: H, + diag_tx: DiagnosticsSender, + exporter: DynExporter, + ) -> Self { + let root = compiler_driver.inner.world.root.clone(); + let driver = CompileExporter::new(compiler_driver).with_exporter(exporter); + let driver = Reporter { + diag_group, + diag_tx, + inner: driver, + cb, + }; + let inner = CompileActor::new(driver, root.as_ref().to_owned()).with_watch(true); + + Self { + inner, + client: TypstClient { + entry: Arc::new(SyncMutex::new(None)), + inner: once_cell::sync::OnceCell::new(), + }, + } + } + + pub fn spawn(self) -> Result, Error> { + let (server, client) = self.inner.split(); + tokio::spawn(server.spawn()); + + self.client.inner.set(client).ok().unwrap(); + + Ok(self.client) + } +} + pub struct Reporter { diag_group: String, diag_tx: DiagnosticsSender, @@ -615,157 +533,15 @@ impl WorldExporter for Reporter { impl, H> Reporter { fn push_diagnostics(&mut self, diagnostics: EcoVec) { - fn convert_diagnostics<'a>( - project: &TypstSystemWorld, - errors: impl IntoIterator, - position_encoding: PositionEncoding, - ) -> DiagnosticsMap { - errors - .into_iter() - .flat_map(|error| { - convert_diagnostic(project, error, position_encoding) - .map_err(move |conversion_err| { - error!("could not convert Typst error to diagnostic: {conversion_err:?} error to convert: {error:?}"); - }) - }) - .collect::>() - .into_iter() - .into_group_map() - } - - fn convert_diagnostic( - project: &TypstSystemWorld, - typst_diagnostic: &TypstDiagnostic, - position_encoding: PositionEncoding, - ) -> anyhow::Result<(Url, LspDiagnostic)> { - let uri; - let lsp_range; - if let Some((id, span)) = diagnostic_span_id(typst_diagnostic) { - uri = Url::from_file_path(project.path_for_id(id)?).unwrap(); - let source = project.source(id)?; - lsp_range = diagnostic_range(&source, span, position_encoding).raw_range; - } else { - uri = Url::from_file_path(project.root.clone()).unwrap(); - lsp_range = LspRawRange::default(); - }; - - let lsp_severity = diagnostic_severity(typst_diagnostic.severity); - - let typst_message = &typst_diagnostic.message; - let typst_hints = &typst_diagnostic.hints; - let lsp_message = format!("{typst_message}{}", diagnostic_hints(typst_hints)); - - let tracepoints = - diagnostic_related_information(project, typst_diagnostic, position_encoding)?; - - let diagnostic = LspDiagnostic { - range: lsp_range, - severity: Some(lsp_severity), - message: lsp_message, - source: Some("typst".to_owned()), - related_information: Some(tracepoints), - ..Default::default() - }; - - Ok((uri, diagnostic)) - } - - fn tracepoint_to_relatedinformation( - project: &TypstSystemWorld, - tracepoint: &Spanned, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - if let Some(id) = tracepoint.span.id() { - let uri = Url::from_file_path(project.path_for_id(id)?).unwrap(); - let source = project.source(id)?; - - if let Some(typst_range) = source.range(tracepoint.span) { - let lsp_range = typst_to_lsp::range(typst_range, &source, position_encoding); - - return Ok(Some(DiagnosticRelatedInformation { - location: LspLocation { - uri, - range: lsp_range.raw_range, - }, - message: tracepoint.v.to_string(), - })); - } - } - - Ok(None) - } - - fn diagnostic_related_information( - project: &TypstSystemWorld, - typst_diagnostic: &TypstDiagnostic, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - let mut tracepoints = vec![]; - - for tracepoint in &typst_diagnostic.trace { - if let Some(info) = - tracepoint_to_relatedinformation(project, tracepoint, position_encoding)? - { - tracepoints.push(info); - } - } - - Ok(tracepoints) - } - - fn diagnostic_span_id(typst_diagnostic: &TypstDiagnostic) -> Option<(FileId, TypstSpan)> { - iter::once(typst_diagnostic.span) - .chain(typst_diagnostic.trace.iter().map(|trace| trace.span)) - .find_map(|span| Some((span.id()?, span))) - } - - fn diagnostic_range( - source: &Source, - typst_span: TypstSpan, - position_encoding: PositionEncoding, - ) -> LspRange { - // Due to #241 and maybe typst/typst#2035, we sometimes fail to find the span. - // In that case, we use a default span as a better alternative to - // panicking. - // - // This may have been fixed after Typst 0.7.0, but it's still nice to avoid - // panics in case something similar reappears. - match source.find(typst_span) { - Some(node) => { - let typst_range = node.range(); - typst_to_lsp::range(typst_range, source, position_encoding) - } - None => LspRange::new( - LspRawRange::new(LspPosition::new(0, 0), LspPosition::new(0, 0)), - position_encoding, - ), - } - } - - fn diagnostic_severity(typst_severity: TypstSeverity) -> LspSeverity { - match typst_severity { - TypstSeverity::Error => LspSeverity::ERROR, - TypstSeverity::Warning => LspSeverity::WARNING, - } - } - - fn diagnostic_hints( - typst_hints: &[EcoString], - ) -> Format + '_> { - iter::repeat(EcoString::from("\n\nHint: ")) - .take(typst_hints.len()) - .interleave(typst_hints.iter().cloned()) - .format("") - } + trace!("send diagnostics: {:#?}", diagnostics); // todo encoding - let diagnostics = convert_diagnostics( + let diagnostics = tinymist_query::convert_diagnostics( self.inner.world(), diagnostics.as_ref(), PositionEncoding::Utf16, ); - trace!("send diagnostics: {:#?}", diagnostics); let err = self.diag_tx.send((self.diag_group.clone(), diagnostics)); if let Err(err) = err { error!("failed to send diagnostics: {:#}", err); @@ -773,43 +549,6 @@ impl, H> Reporter { } } -impl CompileServer { - pub fn new( - diag_group: String, - compiler_driver: CompileDriver, - cb: H, - diag_tx: DiagnosticsSender, - exporter: DynExporter, - ) -> Self { - let root = compiler_driver.inner.world.root.clone(); - let driver = CompileExporter::new(compiler_driver).with_exporter(exporter); - let driver = Reporter { - diag_group, - diag_tx, - inner: driver, - cb, - }; - let inner = CompileActor::new(driver, root.as_ref().to_owned()).with_watch(true); - - Self { - inner, - client: TypstClient { - entry: Arc::new(SyncMutex::new(None)), - inner: once_cell::sync::OnceCell::new(), - }, - } - } - - pub fn spawn(self) -> Result, Error> { - let (server, client) = self.inner.split(); - tokio::spawn(server.spawn()); - - self.client.inner.set(client).ok().unwrap(); - - Ok(self.client) - } -} - pub struct TypstClient { entry: Arc>>, inner: once_cell::sync::OnceCell>, @@ -965,58 +704,20 @@ impl CompileNode { &self, query: CompilerQueryRequest, ) -> anyhow::Result { + use tinymist_query::*; + use CompilerQueryRequest::*; + match query { CompilerQueryRequest::OnSaveExport(OnSaveExportRequest { path }) => { self.on_save_export(path).await?; Ok(CompilerQueryResponse::OnSaveExport(())) } - CompilerQueryRequest::Hover(HoverRequest { - path, - position, - position_encoding, - }) => self - .hover(path, position, position_encoding) - .await - .map(CompilerQueryResponse::Hover), - CompilerQueryRequest::Completion(CompletionRequest { - path, - position, - position_encoding, - explicit, - }) => self - .completion(path, position, position_encoding, explicit) - .await - .map(CompilerQueryResponse::Completion), - CompilerQueryRequest::SignatureHelp(SignatureHelpRequest { - path, - position, - position_encoding, - }) => self - .signature_help(path, position, position_encoding) - .await - .map(CompilerQueryResponse::SignatureHelp), - CompilerQueryRequest::DocumentSymbol(DocumentSymbolRequest { - path, - position_encoding, - }) => self - .document_symbol(path, position_encoding) - .await - .map(CompilerQueryResponse::DocumentSymbol), - CompilerQueryRequest::Symbol(SymbolRequest { - pattern, - position_encoding, - }) => self - .symbol(pattern, position_encoding) - .await - .map(CompilerQueryResponse::Symbol), - CompilerQueryRequest::SelectionRange(SelectionRangeRequest { - path, - positions, - position_encoding, - }) => self - .selection_range(path, positions, position_encoding) - .await - .map(CompilerQueryResponse::SelectionRange), + Hover(req) => query_state!(self, Hover, hover, req), + Completion(req) => query_state!(self, Completion, completion, req), + SignatureHelp(req) => query_world!(self, SignatureHelp, signature_help, req), + DocumentSymbol(req) => query_world!(self, DocumentSymbol, document_symbol, req), + Symbol(req) => query_world!(self, Symbol, symbol, req), + SelectionRange(req) => query_world!(self, SelectionRange, selection_range, req), CompilerQueryRequest::SemanticTokensDelta(..) | CompilerQueryRequest::SemanticTokensFull(..) => unreachable!(), } @@ -1026,491 +727,13 @@ impl CompileNode { Ok(()) } - async fn hover( + async fn steal_world( &self, - path: PathBuf, - position: LspPosition, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - let doc = self.handler.result.lock().unwrap().clone().ok(); - + f: impl FnOnce(&TypstSystemWorld) -> T + Send + Sync + 'static, + ) -> anyhow::Result { let mut client = self.inner.lock().await; - let fut = client.steal_async(move |compiler, _| { - let world = compiler.compiler.world(); - - let source = get_suitable_source_in_workspace(world, &path).ok()?; - let typst_offset = - lsp_to_typst::position_to_offset(position, position_encoding, &source); - - let typst_tooltip = typst_ide::tooltip(world, doc.as_deref(), &source, typst_offset)?; - - let ast_node = LinkedNode::new(source.root()).leaf_at(typst_offset)?; - let range = typst_to_lsp::range(ast_node.range(), &source, position_encoding); - - Some(Hover { - contents: typst_to_lsp::tooltip(&typst_tooltip), - range: Some(range.raw_range), - }) - }); - - Ok(fut.await?) - } - - async fn completion( - &self, - path: PathBuf, - position: LspPosition, - position_encoding: PositionEncoding, - explicit: bool, - ) -> anyhow::Result> { - let doc = self.handler.result.lock().unwrap().clone().ok(); - - let mut client = self.inner.lock().await; - let fut = client.steal_async(move |compiler, _| { - let world = compiler.compiler.world(); - - let source = get_suitable_source_in_workspace(world, &path).ok()?; - let typst_offset = - lsp_to_typst::position_to_offset(position, position_encoding, &source); - - let (typst_start_offset, completions) = - typst_ide::autocomplete(world, doc.as_deref(), &source, typst_offset, explicit)?; - - let lsp_start_position = - typst_to_lsp::offset_to_position(typst_start_offset, position_encoding, &source); - let replace_range = LspRawRange::new(lsp_start_position, position); - Some(typst_to_lsp::completions(&completions, replace_range).into()) - }); - - Ok(fut.await?) - } - - async fn signature_help( - &self, - path: PathBuf, - position: LspPosition, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - fn surrounding_function_syntax<'b>( - leaf: &'b LinkedNode, - ) -> Option<(ast::Expr<'b>, LinkedNode<'b>, ast::Args<'b>)> { - let parent = leaf.parent()?; - let parent = match parent.kind() { - SyntaxKind::Named => parent.parent()?, - _ => parent, - }; - let args = parent.cast::()?; - let grand = parent.parent()?; - let expr = grand.cast::()?; - let callee = match expr { - ast::Expr::FuncCall(call) => call.callee(), - ast::Expr::Set(set) => set.target(), - _ => return None, - }; - Some((callee, grand.find(callee.span())?, args)) - } - - fn param_index_at_leaf( - leaf: &LinkedNode, - function: &Func, - args: ast::Args, - ) -> Option { - let deciding = deciding_syntax(leaf); - let params = function.params()?; - let param_index = find_param_index(&deciding, params, args)?; - trace!("got param index {param_index}"); - Some(param_index) - } - - /// Find the piece of syntax that decides what we're completing. - fn deciding_syntax<'b>(leaf: &'b LinkedNode) -> LinkedNode<'b> { - let mut deciding = leaf.clone(); - while !matches!( - deciding.kind(), - SyntaxKind::LeftParen | SyntaxKind::Comma | SyntaxKind::Colon - ) { - let Some(prev) = deciding.prev_leaf() else { - break; - }; - deciding = prev; - } - deciding - } - - fn find_param_index( - deciding: &LinkedNode, - params: &[ParamInfo], - args: ast::Args, - ) -> Option { - match deciding.kind() { - // After colon: "func(param:|)", "func(param: |)". - SyntaxKind::Colon => { - let prev = deciding.prev_leaf()?; - let param_ident = prev.cast::()?; - params - .iter() - .position(|param| param.name == param_ident.as_str()) - } - // Before: "func(|)", "func(hi|)", "func(12,|)". - SyntaxKind::Comma | SyntaxKind::LeftParen => { - let next = deciding.next_leaf(); - let following_param = next.as_ref().and_then(|next| next.cast::()); - match following_param { - Some(next) => params - .iter() - .position(|param| param.named && param.name.starts_with(next.as_str())), - None => { - let positional_args_so_far = args - .items() - .filter(|arg| matches!(arg, ast::Arg::Pos(_))) - .count(); - params - .iter() - .enumerate() - .filter(|(_, param)| param.positional) - .map(|(i, _)| i) - .nth(positional_args_so_far) - } - } - } - _ => None, - } - } - - fn markdown_docs(docs: &str) -> Documentation { - Documentation::MarkupContent(MarkupContent { - kind: MarkupKind::Markdown, - value: docs.to_owned(), - }) - } - - let mut client = self.inner.lock().await; - let fut = client.steal_async(move |compiler, _| { - let world = compiler.compiler.world(); - - let source = get_suitable_source_in_workspace(world, &path).ok()?; - let typst_offset = - lsp_to_typst::position_to_offset(position, position_encoding, &source); - - let ast_node = LinkedNode::new(source.root()).leaf_at(typst_offset)?; - let (callee, callee_node, args) = surrounding_function_syntax(&ast_node)?; - - let mut ancestor = &ast_node; - while !ancestor.is::() { - ancestor = ancestor.parent()?; - } - - if !callee.hash() && !matches!(callee, ast::Expr::MathIdent(_)) { - return None; - } - - let values = analyze_expr(world, &callee_node); - - let function = values.into_iter().find_map(|v| match v { - Value::Func(f) => Some(f), - _ => None, - })?; - trace!("got function {function:?}"); - - let param_index = param_index_at_leaf(&ast_node, &function, args); - - let label = format!( - "{}({}){}", - function.name().unwrap_or(""), - match function.params() { - Some(params) => params - .iter() - .map(typst_to_lsp::param_info_to_label) - .join(", "), - None => "".to_owned(), - }, - match function.returns() { - Some(returns) => format!("-> {}", typst_to_lsp::cast_info_to_label(returns)), - None => "".to_owned(), - } - ); - let params = function - .params() - .unwrap_or_default() - .iter() - .map(typst_to_lsp::param_info) - .collect(); - trace!("got signature info {label} {params:?}"); - - let documentation = function.docs().map(markdown_docs); - - let active_parameter = param_index.map(|i| i as u32); - - Some(SignatureInformation { - label, - documentation, - parameters: Some(params), - active_parameter, - }) - }); - - let signature = fut.await?; - - Ok(signature.map(|signature| SignatureHelp { - signatures: vec![signature], - active_signature: Some(0), - active_parameter: None, - })) - } - - async fn document_symbol( - &self, - path: PathBuf, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - let mut client = self.inner.lock().await; - let fut = client.steal_async(move |compiler, _| { - let world = compiler.compiler.world(); - - let source = get_suitable_source_in_workspace(world, &path).ok()?; - - let uri = Url::from_file_path(path).unwrap(); - let symbols = get_document_symbols(source, uri, position_encoding); - - symbols.map(DocumentSymbolResponse::Flat) - }); - - Ok(fut.await?) - } - - async fn symbol( - &self, - pattern: Option, - position_encoding: PositionEncoding, - ) -> anyhow::Result>> { - let mut client = self.inner.lock().await; - let fut = client.steal_async(move |compiler, _| { - let world = compiler.compiler.world(); - - // todo: expose source - - let mut symbols = vec![]; - - world.iter_dependencies(&mut |path, _| { - let Ok(source) = get_suitable_source_in_workspace(world, path) else { - return; - }; - let uri = Url::from_file_path(path).unwrap(); - let res = - get_document_symbols(source, uri, position_encoding).and_then(|symbols| { - pattern - .as_ref() - .map(|pattern| filter_document_symbols(symbols, pattern)) - }); - - if let Some(mut res) = res { - symbols.append(&mut res) - } - }); - - Some(symbols) - }); - - Ok(fut.await?) - } - - async fn selection_range( - &self, - path: PathBuf, - positions: Vec, - position_encoding: PositionEncoding, - ) -> anyhow::Result>> { - fn range_for_node( - source: &Source, - position_encoding: PositionEncoding, - node: &LinkedNode, - ) -> SelectionRange { - let range = typst_to_lsp::range(node.range(), source, position_encoding); - SelectionRange { - range: range.raw_range, - parent: node - .parent() - .map(|node| Box::new(range_for_node(source, position_encoding, node))), - } - } - - let mut client = self.inner.lock().await; - let fut = client.steal_async(move |compiler, _| { - let world = compiler.compiler.world(); - - let source = get_suitable_source_in_workspace(world, &path).ok()?; - - let mut ranges = Vec::new(); - for position in positions { - let typst_offset = - lsp_to_typst::position_to_offset(position, position_encoding, &source); - let tree = LinkedNode::new(source.root()); - let leaf = tree.leaf_at(typst_offset)?; - ranges.push(range_for_node(&source, position_encoding, &leaf)); - } - - Some(ranges) - }); + let fut = client.steal_async(move |compiler, _| f(compiler.compiler.world())); Ok(fut.await?) } } - -fn get_suitable_source_in_workspace(w: &TypstSystemWorld, p: &Path) -> FileResult { - // todo: source in packages - let relative_path = p - .strip_prefix(&w.workspace_root()) - .map_err(|_| FileError::NotFound(p.to_owned()))?; - w.source(TypstFileId::new(None, VirtualPath::new(relative_path))) -} - -fn filter_document_symbols( - symbols: Vec, - query_string: &str, -) -> Vec { - symbols - .into_iter() - .filter(|e| e.name.contains(query_string)) - .collect() -} - -#[comemo::memoize] -fn get_document_symbols( - source: Source, - uri: Url, - position_encoding: PositionEncoding, -) -> Option> { - struct DocumentSymbolWorker { - symbols: Vec, - } - - impl DocumentSymbolWorker { - /// Get all symbols for a node recursively. - pub fn get_symbols<'a>( - &mut self, - node: LinkedNode<'a>, - source: &'a Source, - uri: &'a Url, - position_encoding: PositionEncoding, - ) -> anyhow::Result<()> { - let own_symbol = get_ident(&node, source, uri, position_encoding)?; - - for child in node.children() { - self.get_symbols(child, source, uri, position_encoding)?; - } - - if let Some(symbol) = own_symbol { - self.symbols.push(symbol); - } - - Ok(()) - } - } - - /// Get symbol for a leaf node of a valid type, or `None` if the node is an - /// invalid type. - #[allow(deprecated)] - fn get_ident( - node: &LinkedNode, - source: &Source, - uri: &Url, - position_encoding: PositionEncoding, - ) -> anyhow::Result> { - match node.kind() { - SyntaxKind::Label => { - let ast_node = node - .cast::() - .ok_or_else(|| anyhow!("cast to ast node failed: {:?}", node))?; - let name = ast_node.get().to_string(); - let symbol = SymbolInformation { - name, - kind: SymbolKind::CONSTANT, - tags: None, - deprecated: None, // do not use, deprecated, use `tags` instead - location: LspLocation { - uri: uri.clone(), - range: typst_to_lsp::range(node.range(), source, position_encoding) - .raw_range, - }, - container_name: None, - }; - Ok(Some(symbol)) - } - SyntaxKind::Ident => { - let ast_node = node - .cast::() - .ok_or_else(|| anyhow!("cast to ast node failed: {:?}", node))?; - let name = ast_node.get().to_string(); - let Some(parent) = node.parent() else { - return Ok(None); - }; - let kind = match parent.kind() { - // for variable definitions, the Let binding holds an Ident - SyntaxKind::LetBinding => SymbolKind::VARIABLE, - // for function definitions, the Let binding holds a Closure which holds the - // Ident - SyntaxKind::Closure => { - let Some(grand_parent) = parent.parent() else { - return Ok(None); - }; - match grand_parent.kind() { - SyntaxKind::LetBinding => SymbolKind::FUNCTION, - _ => return Ok(None), - } - } - _ => return Ok(None), - }; - let symbol = SymbolInformation { - name, - kind, - tags: None, - deprecated: None, // do not use, deprecated, use `tags` instead - location: LspLocation { - uri: uri.clone(), - range: typst_to_lsp::range(node.range(), source, position_encoding) - .raw_range, - }, - container_name: None, - }; - Ok(Some(symbol)) - } - SyntaxKind::Markup => { - let name = node.get().to_owned().into_text().to_string(); - if name.is_empty() { - return Ok(None); - } - let Some(parent) = node.parent() else { - return Ok(None); - }; - let kind = match parent.kind() { - SyntaxKind::Heading => SymbolKind::NAMESPACE, - _ => return Ok(None), - }; - let symbol = SymbolInformation { - name, - kind, - tags: None, - deprecated: None, // do not use, deprecated, use `tags` instead - location: LspLocation { - uri: uri.clone(), - range: typst_to_lsp::range(node.range(), source, position_encoding) - .raw_range, - }, - container_name: None, - }; - Ok(Some(symbol)) - } - _ => Ok(None), - } - } - - let root = LinkedNode::new(source.root()); - - let mut worker = DocumentSymbolWorker { symbols: vec![] }; - - let res = worker - .get_symbols(root, &source, &uri, position_encoding) - .ok(); - - res.map(|_| worker.symbols) -} diff --git a/crates/tinymist/src/config.rs b/crates/tinymist/src/config.rs deleted file mode 100644 index 8c71c4b6..00000000 --- a/crates/tinymist/src/config.rs +++ /dev/null @@ -1,221 +0,0 @@ -use std::{fmt, path::PathBuf}; - -use anyhow::bail; -use futures::future::BoxFuture; -use itertools::Itertools; -use serde::Deserialize; -use serde_json::{Map, Value}; -use tower_lsp::lsp_types::{self, ConfigurationItem, InitializeParams, PositionEncodingKind}; - -use crate::ext::InitializeParamsExt; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ExperimentalFormatterMode { - #[default] - Off, - On, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum ExportPdfMode { - Never, - #[default] - OnSave, - OnType, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum SemanticTokensMode { - Disable, - #[default] - Enable, -} - -pub type Listener = Box BoxFuture> + Send + Sync>; - -const CONFIG_ITEMS: &[&str] = &[ - "exportPdf", - "rootPath", - "semanticTokens", - "experimentalFormatterMode", -]; - -#[derive(Default)] -pub struct Config { - pub export_pdf: ExportPdfMode, - pub root_path: Option, - pub semantic_tokens: SemanticTokensMode, - pub formatter: ExperimentalFormatterMode, - semantic_tokens_listeners: Vec>, - formatter_listeners: Vec>, -} - -impl Config { - pub fn get_items() -> Vec { - let sections = CONFIG_ITEMS - .iter() - .flat_map(|item| [format!("tinymist.{item}"), item.to_string()]); - - sections - .map(|section| ConfigurationItem { - section: Some(section), - ..Default::default() - }) - .collect() - } - - pub fn values_to_map(values: Vec) -> Map { - let unpaired_values = values - .into_iter() - .tuples() - .map(|(a, b)| if !a.is_null() { a } else { b }); - - CONFIG_ITEMS - .iter() - .map(|item| item.to_string()) - .zip(unpaired_values) - .collect() - } - - pub fn listen_semantic_tokens(&mut self, listener: Listener) { - self.semantic_tokens_listeners.push(listener); - } - - // pub fn listen_formatting(&mut self, listener: - // Listener) { self.formatter_listeners. - // push(listener); } - - pub async fn update(&mut self, update: &Value) -> anyhow::Result<()> { - if let Value::Object(update) = update { - self.update_by_map(update).await - } else { - bail!("got invalid configuration object {update}") - } - } - - pub async fn update_by_map(&mut self, update: &Map) -> anyhow::Result<()> { - let export_pdf = update - .get("exportPdf") - .map(ExportPdfMode::deserialize) - .and_then(Result::ok); - if let Some(export_pdf) = export_pdf { - self.export_pdf = export_pdf; - } - - let root_path = update.get("rootPath"); - if let Some(root_path) = root_path { - if root_path.is_null() { - self.root_path = None; - } - if let Some(root_path) = root_path.as_str().map(PathBuf::from) { - self.root_path = Some(root_path); - } - } - - let semantic_tokens = update - .get("semanticTokens") - .map(SemanticTokensMode::deserialize) - .and_then(Result::ok); - if let Some(semantic_tokens) = semantic_tokens { - for listener in &mut self.semantic_tokens_listeners { - listener(&semantic_tokens).await?; - } - self.semantic_tokens = semantic_tokens; - } - - let formatter = update - .get("experimentalFormatterMode") - .map(ExperimentalFormatterMode::deserialize) - .and_then(Result::ok); - if let Some(formatter) = formatter { - for listener in &mut self.formatter_listeners { - listener(&formatter).await?; - } - self.formatter = formatter; - } - - Ok(()) - } -} - -impl fmt::Debug for Config { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - f.debug_struct("Config") - .field("export_pdf", &self.export_pdf) - .field("formatter", &self.formatter) - .field("semantic_tokens", &self.semantic_tokens) - .field( - "semantic_tokens_listeners", - &format_args!("Vec[len = {}]", self.semantic_tokens_listeners.len()), - ) - .field( - "formatter_listeners", - &format_args!("Vec[len = {}]", self.formatter_listeners.len()), - ) - .finish() - } -} - -/// What counts as "1 character" for string indexing. We should always prefer -/// UTF-8, but support UTF-16 as long as it is standard. For more background on -/// encodings and LSP, try ["The bottom emoji breaks rust-analyzer"](https://fasterthanli.me/articles/the-bottom-emoji-breaks-rust-analyzer), -/// a well-written article on the topic. -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Default)] -pub enum PositionEncoding { - /// "1 character" means "1 UTF-16 code unit" - /// - /// This is the only required encoding for LSPs to support, but it's not a - /// natural one (unless you're working in JS). Prefer UTF-8, and refer - /// to the article linked in the `PositionEncoding` docs for more - /// background. - #[default] - Utf16, - /// "1 character" means "1 byte" - Utf8, -} - -impl From for lsp_types::PositionEncodingKind { - fn from(position_encoding: PositionEncoding) -> Self { - match position_encoding { - PositionEncoding::Utf16 => Self::UTF16, - PositionEncoding::Utf8 => Self::UTF8, - } - } -} - -/// Configuration set at initialization that won't change within a single -/// session -#[derive(Debug)] -pub struct ConstConfig { - pub position_encoding: PositionEncoding, - pub supports_semantic_tokens_dynamic_registration: bool, - pub supports_document_formatting_dynamic_registration: bool, - pub supports_config_change_registration: bool, -} - -impl ConstConfig { - fn choose_encoding(params: &InitializeParams) -> PositionEncoding { - let encodings = params.position_encodings(); - if encodings.contains(&PositionEncodingKind::UTF8) { - PositionEncoding::Utf8 - } else { - PositionEncoding::Utf16 - } - } -} - -impl From<&InitializeParams> for ConstConfig { - fn from(params: &InitializeParams) -> Self { - Self { - position_encoding: Self::choose_encoding(params), - supports_semantic_tokens_dynamic_registration: params - .supports_semantic_tokens_dynamic_registration(), - supports_document_formatting_dynamic_registration: params - .supports_document_formatting_dynamic_registration(), - supports_config_change_registration: params.supports_config_change_registration(), - } - } -} diff --git a/crates/tinymist/src/ext.rs b/crates/tinymist/src/ext.rs deleted file mode 100644 index 5b268b06..00000000 --- a/crates/tinymist/src/ext.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::ffi::OsStr; -use std::path::PathBuf; - -use tower_lsp::lsp_types::{DocumentFormattingClientCapabilities, Url}; -use tower_lsp::lsp_types::{ - InitializeParams, Position, PositionEncodingKind, SemanticTokensClientCapabilities, -}; -use typst::syntax::VirtualPath; - -use crate::config::PositionEncoding; - -pub trait InitializeParamsExt { - fn position_encodings(&self) -> &[PositionEncodingKind]; - fn supports_config_change_registration(&self) -> bool; - fn semantic_tokens_capabilities(&self) -> Option<&SemanticTokensClientCapabilities>; - fn document_formatting_capabilities(&self) -> Option<&DocumentFormattingClientCapabilities>; - fn supports_semantic_tokens_dynamic_registration(&self) -> bool; - fn supports_document_formatting_dynamic_registration(&self) -> bool; - fn root_paths(&self) -> Vec; -} - -static DEFAULT_ENCODING: [PositionEncodingKind; 1] = [PositionEncodingKind::UTF16]; - -impl InitializeParamsExt for InitializeParams { - fn position_encodings(&self) -> &[PositionEncodingKind] { - self.capabilities - .general - .as_ref() - .and_then(|general| general.position_encodings.as_ref()) - .map(|encodings| encodings.as_slice()) - .unwrap_or(&DEFAULT_ENCODING) - } - - fn supports_config_change_registration(&self) -> bool { - self.capabilities - .workspace - .as_ref() - .and_then(|workspace| workspace.configuration) - .unwrap_or(false) - } - - fn semantic_tokens_capabilities(&self) -> Option<&SemanticTokensClientCapabilities> { - self.capabilities - .text_document - .as_ref()? - .semantic_tokens - .as_ref() - } - - fn document_formatting_capabilities(&self) -> Option<&DocumentFormattingClientCapabilities> { - self.capabilities - .text_document - .as_ref()? - .formatting - .as_ref() - } - - fn supports_semantic_tokens_dynamic_registration(&self) -> bool { - self.semantic_tokens_capabilities() - .and_then(|semantic_tokens| semantic_tokens.dynamic_registration) - .unwrap_or(false) - } - - fn supports_document_formatting_dynamic_registration(&self) -> bool { - self.document_formatting_capabilities() - .and_then(|document_format| document_format.dynamic_registration) - .unwrap_or(false) - } - - #[allow(deprecated)] // `self.root_path` is marked as deprecated - fn root_paths(&self) -> Vec { - match self.workspace_folders.as_ref() { - Some(roots) => roots - .iter() - .map(|root| &root.uri) - .map(Url::to_file_path) - .collect::, _>>() - .unwrap(), - None => self - .root_uri - .as_ref() - .map(|uri| uri.to_file_path().unwrap()) - .or_else(|| self.root_path.clone().map(PathBuf::from)) - .into_iter() - .collect(), - } - } -} - -pub trait StrExt { - fn encoded_len(&self, encoding: PositionEncoding) -> usize; -} - -impl StrExt for str { - fn encoded_len(&self, encoding: PositionEncoding) -> usize { - match encoding { - PositionEncoding::Utf8 => self.len(), - PositionEncoding::Utf16 => self.chars().map(char::len_utf16).sum(), - } - } -} - -pub trait VirtualPathExt { - fn with_extension(&self, extension: impl AsRef) -> Self; -} - -impl VirtualPathExt for VirtualPath { - fn with_extension(&self, extension: impl AsRef) -> Self { - Self::new(self.as_rooted_path().with_extension(extension)) - } -} - -pub trait PositionExt { - fn delta(&self, to: &Self) -> PositionDelta; -} - -impl PositionExt for Position { - /// Calculates the delta from `self` to `to`. This is in the `SemanticToken` - /// sense, so the delta's `character` is relative to `self`'s - /// `character` iff `self` and `to` are on the same line. Otherwise, - /// it's relative to the start of the line `to` is on. - fn delta(&self, to: &Self) -> PositionDelta { - let line_delta = to.line - self.line; - let char_delta = if line_delta == 0 { - to.character - self.character - } else { - to.character - }; - - PositionDelta { - delta_line: line_delta, - delta_start: char_delta, - } - } -} - -#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Copy, Clone, Default)] -pub struct PositionDelta { - pub delta_line: u32, - pub delta_start: u32, -} diff --git a/crates/tinymist/src/lsp.rs b/crates/tinymist/src/lsp.rs index 2f2ba5d7..6faf2011 100644 --- a/crates/tinymist/src/lsp.rs +++ b/crates/tinymist/src/lsp.rs @@ -1,6 +1,8 @@ pub use tower_lsp::Client as LspHost; use std::borrow::Cow; +use std::fmt; +use std::path::PathBuf; use std::sync::Arc; use anyhow::Context; @@ -9,28 +11,187 @@ use futures::FutureExt; use log::{error, info, trace}; use once_cell::sync::OnceCell; use serde_json::Value as JsonValue; +use tinymist_query::{ + get_semantic_tokens_options, get_semantic_tokens_registration, + get_semantic_tokens_unregistration, CompletionRequest, DocumentSymbolRequest, HoverRequest, + PositionEncoding, SelectionRangeRequest, SemanticTokensDeltaRequest, SemanticTokensFullRequest, + SignatureHelpRequest, SymbolRequest, +}; + +use anyhow::bail; +use futures::future::BoxFuture; +use itertools::Itertools; +use serde::Deserialize; +use serde_json::{Map, Value}; use tokio::sync::{Mutex, RwLock}; -use tower_lsp::lsp_types::*; -use tower_lsp::{jsonrpc, LanguageServer}; +use tower_lsp::lsp_types::ConfigurationItem; +use tower_lsp::{jsonrpc, lsp_types::*, LanguageServer}; use typst::model::Document; use typst_ts_core::config::CompileOpts; use crate::actor; use crate::actor::typst::CompileCluster; -use crate::actor::typst::{ - CompilerQueryResponse, CompletionRequest, DocumentSymbolRequest, HoverRequest, - OnSaveExportRequest, SelectionRangeRequest, SemanticTokensDeltaRequest, - SemanticTokensFullRequest, SignatureHelpRequest, SymbolRequest, -}; -use crate::config::{ - Config, ConstConfig, ExperimentalFormatterMode, ExportPdfMode, SemanticTokensMode, -}; -use crate::ext::InitializeParamsExt; +use crate::actor::typst::{CompilerQueryResponse, OnSaveExportRequest}; -use super::semantic_tokens::{ - get_semantic_tokens_options, get_semantic_tokens_registration, - get_semantic_tokens_unregistration, -}; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExperimentalFormatterMode { + #[default] + Off, + On, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ExportPdfMode { + Never, + #[default] + OnSave, + OnType, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SemanticTokensMode { + Disable, + #[default] + Enable, +} + +pub type Listener = Box BoxFuture> + Send + Sync>; + +const CONFIG_ITEMS: &[&str] = &[ + "exportPdf", + "rootPath", + "semanticTokens", + "experimentalFormatterMode", +]; + +#[derive(Default)] +pub struct Config { + pub export_pdf: ExportPdfMode, + pub root_path: Option, + pub semantic_tokens: SemanticTokensMode, + pub formatter: ExperimentalFormatterMode, + semantic_tokens_listeners: Vec>, + formatter_listeners: Vec>, +} + +impl Config { + pub fn get_items() -> Vec { + let sections = CONFIG_ITEMS + .iter() + .flat_map(|item| [format!("tinymist.{item}"), item.to_string()]); + + sections + .map(|section| ConfigurationItem { + section: Some(section), + ..Default::default() + }) + .collect() + } + + pub fn values_to_map(values: Vec) -> Map { + let unpaired_values = values + .into_iter() + .tuples() + .map(|(a, b)| if !a.is_null() { a } else { b }); + + CONFIG_ITEMS + .iter() + .map(|item| item.to_string()) + .zip(unpaired_values) + .collect() + } + + pub fn listen_semantic_tokens(&mut self, listener: Listener) { + self.semantic_tokens_listeners.push(listener); + } + + // pub fn listen_formatting(&mut self, listener: + // Listener) { self.formatter_listeners. + // push(listener); } + + pub async fn update(&mut self, update: &Value) -> anyhow::Result<()> { + if let Value::Object(update) = update { + self.update_by_map(update).await + } else { + bail!("got invalid configuration object {update}") + } + } + + pub async fn update_by_map(&mut self, update: &Map) -> anyhow::Result<()> { + let export_pdf = update + .get("exportPdf") + .map(ExportPdfMode::deserialize) + .and_then(Result::ok); + if let Some(export_pdf) = export_pdf { + self.export_pdf = export_pdf; + } + + let root_path = update.get("rootPath"); + if let Some(root_path) = root_path { + if root_path.is_null() { + self.root_path = None; + } + if let Some(root_path) = root_path.as_str().map(PathBuf::from) { + self.root_path = Some(root_path); + } + } + + let semantic_tokens = update + .get("semanticTokens") + .map(SemanticTokensMode::deserialize) + .and_then(Result::ok); + if let Some(semantic_tokens) = semantic_tokens { + for listener in &mut self.semantic_tokens_listeners { + listener(&semantic_tokens).await?; + } + self.semantic_tokens = semantic_tokens; + } + + let formatter = update + .get("experimentalFormatterMode") + .map(ExperimentalFormatterMode::deserialize) + .and_then(Result::ok); + if let Some(formatter) = formatter { + for listener in &mut self.formatter_listeners { + listener(&formatter).await?; + } + self.formatter = formatter; + } + + Ok(()) + } +} + +impl fmt::Debug for Config { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("Config") + .field("export_pdf", &self.export_pdf) + .field("formatter", &self.formatter) + .field("semantic_tokens", &self.semantic_tokens) + .field( + "semantic_tokens_listeners", + &format_args!("Vec[len = {}]", self.semantic_tokens_listeners.len()), + ) + .field( + "formatter_listeners", + &format_args!("Vec[len = {}]", self.formatter_listeners.len()), + ) + .finish() + } +} + +/// Configuration set at initialization that won't change within a single +/// session +#[derive(Debug)] +pub struct ConstConfig { + pub position_encoding: PositionEncoding, + pub supports_semantic_tokens_dynamic_registration: bool, + pub supports_document_formatting_dynamic_registration: bool, + pub supports_config_change_registration: bool, +} pub struct TypstServer { pub client: LspHost, @@ -583,3 +744,105 @@ impl TypstServer { todo!() } } + +impl ConstConfig { + fn choose_encoding(params: &InitializeParams) -> PositionEncoding { + let encodings = params.position_encodings(); + if encodings.contains(&PositionEncodingKind::UTF8) { + PositionEncoding::Utf8 + } else { + PositionEncoding::Utf16 + } + } +} + +impl From<&InitializeParams> for ConstConfig { + fn from(params: &InitializeParams) -> Self { + Self { + position_encoding: Self::choose_encoding(params), + supports_semantic_tokens_dynamic_registration: params + .supports_semantic_tokens_dynamic_registration(), + supports_document_formatting_dynamic_registration: params + .supports_document_formatting_dynamic_registration(), + supports_config_change_registration: params.supports_config_change_registration(), + } + } +} + +pub trait InitializeParamsExt { + fn position_encodings(&self) -> &[PositionEncodingKind]; + fn supports_config_change_registration(&self) -> bool; + fn semantic_tokens_capabilities(&self) -> Option<&SemanticTokensClientCapabilities>; + fn document_formatting_capabilities(&self) -> Option<&DocumentFormattingClientCapabilities>; + fn supports_semantic_tokens_dynamic_registration(&self) -> bool; + fn supports_document_formatting_dynamic_registration(&self) -> bool; + fn root_paths(&self) -> Vec; +} + +static DEFAULT_ENCODING: [PositionEncodingKind; 1] = [PositionEncodingKind::UTF16]; + +impl InitializeParamsExt for InitializeParams { + fn position_encodings(&self) -> &[PositionEncodingKind] { + self.capabilities + .general + .as_ref() + .and_then(|general| general.position_encodings.as_ref()) + .map(|encodings| encodings.as_slice()) + .unwrap_or(&DEFAULT_ENCODING) + } + + fn supports_config_change_registration(&self) -> bool { + self.capabilities + .workspace + .as_ref() + .and_then(|workspace| workspace.configuration) + .unwrap_or(false) + } + + fn semantic_tokens_capabilities(&self) -> Option<&SemanticTokensClientCapabilities> { + self.capabilities + .text_document + .as_ref()? + .semantic_tokens + .as_ref() + } + + fn document_formatting_capabilities(&self) -> Option<&DocumentFormattingClientCapabilities> { + self.capabilities + .text_document + .as_ref()? + .formatting + .as_ref() + } + + fn supports_semantic_tokens_dynamic_registration(&self) -> bool { + self.semantic_tokens_capabilities() + .and_then(|semantic_tokens| semantic_tokens.dynamic_registration) + .unwrap_or(false) + } + + fn supports_document_formatting_dynamic_registration(&self) -> bool { + self.document_formatting_capabilities() + .and_then(|document_format| document_format.dynamic_registration) + .unwrap_or(false) + } + + #[allow(deprecated)] // `self.root_path` is marked as deprecated + fn root_paths(&self) -> Vec { + match self.workspace_folders.as_ref() { + Some(roots) => roots + .iter() + .map(|root| &root.uri) + .map(Url::to_file_path) + .collect::, _>>() + .unwrap(), + None => self + .root_uri + .as_ref() + .map(|uri| uri.to_file_path().unwrap()) + .or_else(|| self.root_path.clone().map(PathBuf::from)) + .into_iter() + .collect(), + } + } +} diff --git a/crates/tinymist/src/main.rs b/crates/tinymist/src/main.rs index 897db32b..d65cc9a4 100644 --- a/crates/tinymist/src/main.rs +++ b/crates/tinymist/src/main.rs @@ -1,19 +1,11 @@ //! # tinymist LSP Server -mod config; -mod ext; -mod lsp_typst_boundary; - // pub mod formatting; pub mod actor; -pub mod analysis; pub mod lsp; -pub mod semantic_tokens; use tower_lsp::{LspService, Server}; -use lsp::TypstServer; - // #[derive(Debug, Clone)] // struct Args {} @@ -49,7 +41,7 @@ async fn main() { let stdin = tokio::io::stdin(); let stdout = tokio::io::stdout(); - let (service, socket) = LspService::new(TypstServer::new); + let (service, socket) = LspService::new(lsp::TypstServer::new); Server::new(stdin, stdout, socket).serve(service).await; } diff --git a/crates/tinymist/src/semantic_tokens/token_encode.rs b/crates/tinymist/src/semantic_tokens/token_encode.rs deleted file mode 100644 index 455f2bf8..00000000 --- a/crates/tinymist/src/semantic_tokens/token_encode.rs +++ /dev/null @@ -1,44 +0,0 @@ -use tower_lsp::lsp_types::{Position, SemanticToken}; -use typst::diag::EcoString; -use typst::syntax::Source; - -use crate::config::PositionEncoding; -use crate::ext::{PositionExt, StrExt}; -use crate::lsp_typst_boundary::typst_to_lsp; - -use super::Token; - -pub(super) fn encode_tokens<'a>( - tokens: impl Iterator + 'a, - source: &'a Source, - encoding: PositionEncoding, -) -> impl Iterator + 'a { - tokens.scan(Position::new(0, 0), move |last_position, token| { - let (encoded_token, source_code, position) = - encode_token(token, last_position, source, encoding); - *last_position = position; - Some((encoded_token, source_code)) - }) -} - -fn encode_token( - token: Token, - last_position: &Position, - source: &Source, - encoding: PositionEncoding, -) -> (SemanticToken, EcoString, Position) { - let position = typst_to_lsp::offset_to_position(token.offset, encoding, source); - let delta = last_position.delta(&position); - - let length = token.source.as_str().encoded_len(encoding); - - let lsp_token = SemanticToken { - delta_line: delta.delta_line, - delta_start: delta.delta_start, - length: length as u32, - token_type: token.token_type as u32, - token_modifiers_bitset: token.modifiers.bitset(), - }; - - (lsp_token, token.source, position) -}