refactor: build and move comment and matcher crates to analysis crate (#1223)

crates/tinymist-analysis/Cargo.toml

@@ -12,10 +12,12 @@ repository.workspace = true
 rust-version.workspace = true
 
 [dependencies]
+ecow.workspace = true
+lsp-types.workspace = true
+serde.workspace = true
+strum.workspace = true
 toml.workspace = true
 typst.workspace = true
-serde.workspace = true
-lsp-types.workspace = true
 
 [dev-dependencies]
 insta.workspace = true

crates/tinymist-analysis/src/lib.rs

@@ -1,5 +1,21 @@
 //! Tinymist Analysis
 
 pub mod debug_loc;
-pub mod import;
 mod prelude;
+pub mod syntax;
+
+/// Completely disabled log
+#[macro_export]
+macro_rules! log_debug_ct_ {
+    // debug!(target: "my_target", key1 = 42, key2 = true; "a {} event", "log")
+    // debug!(target: "my_target", "a {} event", "log")
+    (target: $target:expr, $($arg:tt)+) => {
+        let _ = format_args!($target, $($arg)+);
+    };
+
+    // debug!("a {} event", "log")
+    ($($arg:tt)+) => {
+        let _ = format_args!($($arg)+);
+    };
+}
+pub use log_debug_ct_ as log_debug_ct;

crates/tinymist-analysis/src/prelude.rs

@@ -1,10 +1,12 @@
+pub use std::ops::Range;
 pub use std::path::Path;
 
-pub use typst::diag::FileError;
+pub use ecow::EcoVec;
+pub use typst::diag::{EcoString, FileError};
 pub use typst::syntax::FileId as TypstFileId;
 pub use typst::syntax::{
-    ast::{self},
+    ast::{self, AstNode},
     package::{PackageManifest, PackageSpec},
-    Source, VirtualPath,
+    LinkedNode, Source, SyntaxKind, SyntaxNode, VirtualPath,
 };
 pub use typst::World;

crates/tinymist-analysis/src/syntax.rs

@@ -0,0 +1,10 @@
+//! Analyzing the syntax of a source file.
+//!
+//! This module must hide all **AST details** from the rest of the codebase.
+
+pub mod import;
+pub use import::*;
+pub mod comment;
+pub use comment::*;
+pub mod matcher;
+pub use matcher::*;

crates/tinymist-analysis/src/syntax/comment.rs

@@ -1,5 +1,8 @@
+//! Convenient utilities to match comment in code.
+
 use crate::prelude::*;
 
+/// Extract the module-level documentation from a source.
 pub fn find_module_level_docs(src: &Source) -> Option<String> {
     crate::log_debug_ct!("finding docs at: {id:?}", id = src.id());
 
@@ -46,20 +49,38 @@ fn extract_mod_docs_between(
     matcher.collect()
 }
 
+/// A signal raised by the comment group matcher.
 pub enum CommentGroupSignal {
+    /// A hash marker is found.
     Hash,
+    /// A space is found.
     Space,
+    /// A line comment is found.
     LineComment,
+    /// A block comment is found.
     BlockComment,
+    /// The comment group should be broken.
     BreakGroup,
 }
 
+/// A matcher that groups comments.
 #[derive(Default)]
 pub struct CommentGroupMatcher {
     newline_count: u32,
 }
 
 impl CommentGroupMatcher {
+    /// Reset the matcher. This usually happens after a group is collected or
+    /// when some other child item is breaking the comment group manually.
+    pub fn reset(&mut self) {
+        self.newline_count = 0;
+    }
+
+    /// Process a child relative to some [`SyntaxNode`].
+    ///
+    /// ## Example
+    ///
+    /// See [`DocCommentMatcher`] for a real-world example.
     pub fn process(&mut self, n: &SyntaxNode) -> CommentGroupSignal {
         match n.kind() {
             SyntaxKind::Hash => {
@@ -95,16 +116,13 @@ impl CommentGroupMatcher {
             }
         }
     }
-
-    pub fn reset(&mut self) {
-        self.newline_count = 0;
-    }
 }
 enum RawComment {
     Line(EcoString),
     Block(EcoString),
 }
 
+/// A matcher that collects documentation comments.
 #[derive(Default)]
 pub struct DocCommentMatcher {
     comments: Vec<RawComment>,
@@ -113,6 +131,14 @@ pub struct DocCommentMatcher {
 }
 
 impl DocCommentMatcher {
+    /// Reset the matcher. This usually happens after a group is collected or
+    /// when some other child item is breaking the comment group manually.
+    pub fn reset(&mut self) {
+        self.comments.clear();
+        self.group_matcher.reset();
+    }
+
+    /// Process a child relative to some [`SyntaxNode`].
     pub fn process(&mut self, n: &SyntaxNode) -> bool {
         match self.group_matcher.process(n) {
             CommentGroupSignal::LineComment => {
@@ -136,6 +162,7 @@ impl DocCommentMatcher {
         false
     }
 
+    /// Collect the comments and return the result.
     pub fn collect(&mut self) -> Option<String> {
         let comments = &self.comments;
         if comments.is_empty() {
@@ -185,9 +212,4 @@ impl DocCommentMatcher {
         self.comments.clear();
         res
     }
-
-    pub fn reset(&mut self) {
-        self.comments.clear();
-        self.group_matcher.reset();
-    }
 }

crates/tinymist-analysis/src/syntax/matcher.rs

@@ -56,8 +56,8 @@
 //!                      ^ SurroundingSyntax::Regular
 //! ```
 
+use crate::debug_loc::SourceSpanOffset;
 use serde::{Deserialize, Serialize};
-use tinymist_std::debug_loc::SourceSpanOffset;
 use typst::syntax::Span;
 
 use crate::prelude::*;
@@ -84,6 +84,7 @@ pub enum PreviousItem<'a> {
 }
 
 impl<'a> PreviousItem<'a> {
+    /// Gets the underlying [`LinkedNode`] of the item.
     pub fn node(&self) -> &'a LinkedNode<'a> {
         match self {
             PreviousItem::Sibling(node) => node,
@@ -124,9 +125,38 @@ pub fn previous_items<T>(
     None
 }
 
+/// A declaration that is an ancestor of the given node or the previous sibling
+/// of some ancestor.
 pub enum PreviousDecl<'a> {
+    /// An declaration having an identifier.
+    ///
+    /// ## Example
+    ///
+    /// The `x` in the following code:
+    ///
+    /// ```typst
+    /// #let x = 1;
+    /// ```
     Ident(ast::Ident<'a>),
+    /// An declaration yielding from an import source.
+    ///
+    /// ## Example
+    ///
+    /// The `x` in the following code:
+    ///
+    /// ```typst
+    /// #import "path.typ": x;
+    /// ```
     ImportSource(ast::Expr<'a>),
+    /// A wildcard import that possibly containing visible declarations.
+    ///
+    /// ## Example
+    ///
+    /// The following import is matched:
+    ///
+    /// ```typst
+    /// #import "path.typ": *;
+    /// ```
     ImportAll(ast::ModuleImport<'a>),
 }
 
@@ -458,7 +488,26 @@ fn adjust_expr(mut node: LinkedNode) -> Option<LinkedNode> {
 /// Classes of field syntax that can be operated on by IDE functionality.
 #[derive(Debug, Clone)]
 pub enum FieldClass<'a> {
+    /// A field node.
+    ///
+    /// ## Example
+    ///
+    /// The `x` in the following code:
+    ///
+    /// ```typst
+    /// #a.x
+    /// ```
     Field(LinkedNode<'a>),
+
+    /// A dot suffix missing a field.
+    ///
+    /// ## Example
+    ///
+    /// The `.` in the following code:
+    ///
+    /// ```typst
+    /// #a.
+    /// ```
     DotSuffix(SourceSpanOffset),
 }
 
@@ -560,7 +609,9 @@ pub enum SyntaxClass<'a> {
     VarAccess(VarClass<'a>),
     /// A (content) label expression.
     Label {
+        /// The node of the label.
         node: LinkedNode<'a>,
+        /// Whether the label is converted from an error node.
         is_error: bool,
     },
     /// A (content) reference expression.
@@ -734,8 +785,11 @@ fn possible_in_code_trivia(kind: SyntaxKind) -> bool {
 pub enum ArgClass<'a> {
     /// A positional argument.
     Positional {
+        /// The spread arguments met before the positional argument.
         spreads: EcoVec<LinkedNode<'a>>,
+        /// The index of the positional argument.
         positional: usize,
+        /// Whether the positional argument is a spread argument.
         is_spread: bool,
     },
     /// A named argument.
@@ -744,7 +798,7 @@ pub enum ArgClass<'a> {
 
 impl ArgClass<'_> {
     /// Creates the class refer to the first positional argument.
-    pub(crate) fn first_positional() -> Self {
+    pub fn first_positional() -> Self {
         ArgClass::Positional {
             spreads: EcoVec::new(),
             positional: 0,
@@ -754,17 +808,26 @@ impl ArgClass<'_> {
 }
 
 // todo: whether we can merge `SurroundingSyntax` and `SyntaxContext`?
+/// Classes of syntax context (outer syntax) that can be operated on by IDE
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash, strum::EnumIter)]
 pub enum SurroundingSyntax {
+    /// Regular syntax.
     Regular,
+    /// Content in a string.
     StringContent,
+    /// The cursor is directly on the selector of a show rule.
     Selector,
+    /// The cursor is directly on the transformation of a show rule.
     ShowTransform,
+    /// The cursor is directly on the import list.
     ImportList,
+    /// The cursor is directly on the set rule.
     SetRule,
+    /// The cursor is directly on the parameter list.
     ParamList,
 }
 
+/// Determines the surrounding syntax of the node at the position.
 pub fn surrounding_syntax(node: &LinkedNode) -> SurroundingSyntax {
     check_previous_syntax(node)
         .or_else(|| check_surrounding_syntax(node))
@@ -907,19 +970,28 @@ fn enclosed_by(parent: &LinkedNode, s: Option<Span>, leaf: &LinkedNode) -> bool
 pub enum SyntaxContext<'a> {
     /// A cursor on an argument.
     Arg {
+        /// The callee node.
         callee: LinkedNode<'a>,
+        /// The arguments node.
         args: LinkedNode<'a>,
+        /// The argument target pointed by the cursor.
         target: ArgClass<'a>,
+        /// Whether the callee is a set rule.
         is_set: bool,
     },
     /// A cursor on an element in an array or dictionary literal.
     Element {
+        /// The container node.
         container: LinkedNode<'a>,
+        /// The element target pointed by the cursor.
         target: ArgClass<'a>,
     },
     /// A cursor on a parenthesized expression.
     Paren {
+        /// The parenthesized expression node.
         container: LinkedNode<'a>,
+        /// Whether the cursor is on the left side of the parenthesized
+        /// expression.
         is_before: bool,
     },
     /// A variable access expression.
@@ -932,7 +1004,9 @@ pub enum SyntaxContext<'a> {
     IncludePath(LinkedNode<'a>),
     /// A cursor on a label.
     Label {
+        /// The label node.
         node: LinkedNode<'a>,
+        /// Whether the label is converted from an error node.
         is_error: bool,
     },
     /// A cursor on a normal [`SyntaxClass`].
@@ -1201,8 +1275,7 @@ fn arg_context<'a>(
 mod tests {
     use super::*;
     use insta::assert_snapshot;
-    use typst::syntax::{is_newline, Source};
-    use typst_shim::syntax::LinkedNodeExt;
+    use typst::syntax::{is_newline, Side, Source};
 
     fn map_node(source: &str, mapper: impl Fn(&LinkedNode, usize) -> char) -> String {
         let source = Source::detached(source.to_owned());
@@ -1230,7 +1303,7 @@ mod tests {
 
     fn map_syntax(source: &str) -> String {
         map_node(source, |root, cursor| {
-            let node = root.leaf_at_compat(cursor);
+            let node = root.leaf_at(cursor, Side::Before);
             let kind = node.and_then(|node| classify_syntax(node, cursor));
             match kind {
                 Some(SyntaxClass::VarAccess(..)) => 'v',
@@ -1247,7 +1320,7 @@ mod tests {
 
     fn map_context(source: &str) -> String {
         map_node(source, |root, cursor| {
-            let node = root.leaf_at_compat(cursor);
+            let node = root.leaf_at(cursor, Side::Before);
             let kind = node.and_then(|node| classify_context(node, Some(cursor)));
             match kind {
                 Some(SyntaxContext::Arg { .. }) => 'p',
@@ -1353,7 +1426,7 @@ Text
             };
             let source = Source::detached(s.to_owned());
             let root = LinkedNode::new(source.root());
-            let node = root.leaf_at_compat(cursor as usize)?;
+            let node = root.leaf_at(cursor as usize, Side::Before)?;
             let syntax = classify_syntax(node, cursor as usize)?;
             let SyntaxClass::VarAccess(var) = syntax else {
                 return None;

crates/tinymist-query/src/lib.rs

@@ -135,20 +135,7 @@ pub trait StatefulRequest {
     ) -> Option<Self::Response>;
 }
 
-/// Completely disabled log
-#[macro_export]
-macro_rules! log_debug_ct {
-    // debug!(target: "my_target", key1 = 42, key2 = true; "a {} event", "log")
-    // debug!(target: "my_target", "a {} event", "log")
-    (target: $target:expr, $($arg:tt)+) => {
-        let _ = format_args!($target, $($arg)+);
-    };
-
-    // debug!("a {} event", "log")
-    ($($arg:tt)+) => {
-        let _ = format_args!($($arg)+);
-    };
-}
+use tinymist_analysis::log_debug_ct;
 
 #[allow(missing_docs)]
 mod polymorphic {

crates/tinymist-query/src/syntax/expr.rs

@@ -4,7 +4,6 @@ use parking_lot::Mutex;
 use rpds::RedBlackTreeMapSync;
 use rustc_hash::FxHashMap;
 use std::ops::Deref;
-use tinymist_analysis::import::resolve_id_by_path;
 use tinymist_std::hash::hash128;
 use typst::{
     foundations::{Element, NativeElement, Value},
@@ -16,7 +15,7 @@ use typst::{
 use crate::{
     analysis::{QueryStatGuard, SharedContext},
     prelude::*,
-    syntax::{find_module_level_docs, DefKind},
+    syntax::{find_module_level_docs, resolve_id_by_path, DefKind},
     ty::{BuiltinTy, InsTy, Interned, Ty},
 };
 

crates/tinymist-query/src/syntax/mod.rs

@@ -5,15 +5,13 @@
 // todo: remove this
 #![allow(missing_docs)]
 
-pub use tinymist_analysis::import::*;
+pub use tinymist_analysis::syntax::comment::*;
+pub use tinymist_analysis::syntax::import::*;
+pub use tinymist_analysis::syntax::matcher::*;
 pub(crate) mod lexical_hierarchy;
 pub use lexical_hierarchy::*;
-pub mod matcher;
-pub use matcher::*;
 pub(crate) mod module;
 pub use module::*;
-pub(crate) mod comment;
-pub use comment::*;
 pub(crate) mod expr;
 pub use expr::*;
 pub(crate) mod docs;

crates/typlite/src/lib.rs

@@ -710,7 +710,7 @@ impl TypliteWorker {
 
         let path = include.source();
         let src =
-            tinymist_analysis::import::find_source_by_expr(self.world.as_ref(), self.current, path)
+            tinymist_analysis::syntax::find_source_by_expr(self.world.as_ref(), self.current, path)
                 .ok_or_else(|| format!("failed to find source on path {path:?}"))?;
 
         self.clone().sub_file(src).map(Value::Content)