feat: introspect and show complation statistics happening in the language server (#1958)

Adds capability to introspect complations happening in the language
server, to help improve efficiency. I expect most compilations are
caused by tracing for analyzing dynamic expressions, but I haven't
really profiled a document. Then introspection will help confirm or
refute the expectation.

crates/tinymist-analysis/src/stats.rs

@@ -1,6 +1,13 @@
 //! Tinymist Analysis Statistics
 
+use std::fmt::Write;
 use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, LazyLock};
+
+use parking_lot::Mutex;
+use tinymist_std::hash::FxDashMap;
+use tinymist_std::time::Duration;
+use typst::syntax::FileId;
 
 /// Statistics about the allocation
 
@@ -68,6 +75,131 @@ table.alloc-stats tr:nth-child(odd) { background-color: rgba(242, 242, 242, 0.8)
     }
 }
 
+/// The data of the query statistic.
+#[derive(Clone)]
+pub struct QueryStatBucketData {
+    pub(crate) query: u64,
+    pub(crate) missing: u64,
+    pub(crate) total: Duration,
+    pub(crate) min: Duration,
+    pub(crate) max: Duration,
+}
+
+impl Default for QueryStatBucketData {
+    fn default() -> Self {
+        Self {
+            query: 0,
+            missing: 0,
+            total: Duration::from_secs(0),
+            min: Duration::from_secs(u64::MAX),
+            max: Duration::from_secs(0),
+        }
+    }
+}
+
+/// Statistics about some query
+#[derive(Default, Clone)]
+pub struct QueryStatBucket {
+    /// The data of the query statistic.
+    pub data: Arc<Mutex<QueryStatBucketData>>,
+}
+
+/// A guard for the query statistic.
+pub struct QueryStatGuard {
+    /// The bucket of the query statistic.
+    pub bucket: QueryStatBucket,
+    /// The start time of the query.
+    pub since: tinymist_std::time::Instant,
+}
+
+impl Drop for QueryStatGuard {
+    fn drop(&mut self) {
+        let elapsed = self.since.elapsed();
+        let mut data = self.bucket.data.lock();
+        data.query += 1;
+        data.total += elapsed;
+        data.min = data.min.min(elapsed);
+        data.max = data.max.max(elapsed);
+    }
+}
+
+impl QueryStatGuard {
+    /// Increment the missing count.
+    pub fn miss(&self) {
+        let mut data = self.bucket.data.lock();
+        data.missing += 1;
+    }
+}
+
+/// Statistics about the analyzers
+#[derive(Default)]
+pub struct AnalysisStats {
+    /// The query statistics.
+    pub query_stats: FxDashMap<FileId, FxDashMap<&'static str, QueryStatBucket>>,
+}
+
+impl AnalysisStats {
+    /// Gets a statistic guard for a query.
+    pub fn stat(&self, id: FileId, query: &'static str) -> QueryStatGuard {
+        let stats = &self.query_stats;
+        let entry = stats.entry(id).or_default();
+        let entry = entry.entry(query).or_default();
+        QueryStatGuard {
+            bucket: entry.clone(),
+            since: tinymist_std::time::Instant::now(),
+        }
+    }
+
+    /// Reports the statistics of the analysis.
+    pub fn report(&self) -> String {
+        let stats = &self.query_stats;
+        let mut data = Vec::new();
+        for refs in stats.iter() {
+            let id = refs.key();
+            let queries = refs.value();
+            for refs2 in queries.iter() {
+                let query = refs2.key();
+                let bucket = refs2.value().data.lock().clone();
+                let name = format!("{id:?}:{query}").replace('\\', "/");
+                data.push((name, bucket));
+            }
+        }
+
+        // sort by query duration
+        data.sort_by(|x, y| y.1.max.cmp(&x.1.max));
+
+        // format to html
+
+        let mut html = String::new();
+        html.push_str(r#"<div>
+<style>
+table.analysis-stats { width: 100%; border-collapse: collapse; }
+table.analysis-stats th, table.analysis-stats td { border: 1px solid black; padding: 8px; text-align: center; }
+table.analysis-stats th.name-column, table.analysis-stats td.name-column { text-align: left; }
+table.analysis-stats tr:nth-child(odd) { background-color: rgba(242, 242, 242, 0.8); }
+@media (prefers-color-scheme: dark) {
+    table.analysis-stats tr:nth-child(odd) { background-color: rgba(50, 50, 50, 0.8); }
+}
+</style>
+<table class="analysis-stats"><tr><th class="query-column">Name</th><th>Count</th><th>Missing</th><th>Total</th><th>Min</th><th>Max</th></tr>"#);
+
+        for (name, bucket) in data {
+            let _ = write!(
+                &mut html,
+                "<tr><td class=\"query-column\">{name}</td><td>{}</td><td>{}</td><td>{:?}</td><td>{:?}</td><td>{:?}</td></tr>",
+                bucket.query, bucket.missing, bucket.total, bucket.min, bucket.max
+            );
+        }
+        html.push_str("</table>");
+        html.push_str("</div>");
+
+        html
+    }
+}
+
+/// The global statistics about the analyzers.
+pub static GLOBAL_STATS: LazyLock<AnalysisStats> = LazyLock::new(AnalysisStats::default);
+
 fn human_size(size: usize) -> String {
     let units = ["B", "KB", "MB", "GB", "TB"];
     let mut unit = 0;

crates/tinymist-analysis/src/track_values.rs

@@ -3,6 +3,7 @@
 use comemo::Track;
 use ecow::*;
 use tinymist_std::typst::{TypstDocument, TypstPagedDocument};
+use tinymist_world::DETACHED_ENTRY;
 use typst::World;
 use typst::engine::{Engine, Route, Sink, Traced};
 use typst::foundations::{Context, Label, Scopes, Styles, Value};
@@ -11,6 +12,8 @@ use typst::model::BibliographyElem;
 use typst::syntax::{LinkedNode, Span, SyntaxKind, SyntaxNode, ast};
 use typst_shim::eval::Vm;
 
+use crate::stats::GLOBAL_STATS;
+
 /// Try to determine a set of possible values for an expression.
 pub fn analyze_expr(world: &dyn World, node: &LinkedNode) -> EcoVec<(Value, Option<Styles>)> {
     if let Some(parent) = node.parent()
@@ -45,6 +48,8 @@ pub fn analyze_expr_(world: &dyn World, node: &SyntaxNode) -> EcoVec<(Value, Opt
                 return analyze_expr_(world, child);
             }
 
+            let id = node.span().id().unwrap_or_else(|| *DETACHED_ENTRY);
+            let _guard = GLOBAL_STATS.stat(id, "analyze_expr");
             return typst::trace::<TypstPagedDocument>(world, node.span());
         }
     };
@@ -63,6 +68,9 @@ pub fn analyze_import_(world: &dyn World, source: &SyntaxNode) -> (Option<Value>
         return (Some(source.clone()), Some(source));
     }
 
+    let id = source_span.id().unwrap_or_else(|| *DETACHED_ENTRY);
+    let _guard = GLOBAL_STATS.stat(id, "analyze_import");
+
     let introspector = Introspector::default();
     let traced = Traced::default();
     let mut sink = Sink::new();
@@ -116,6 +124,8 @@ pub struct DynLabel {
 pub fn analyze_labels(document: &TypstDocument) -> (Vec<DynLabel>, usize) {
     let mut output = vec![];
 
+    let _guard = GLOBAL_STATS.stat(*DETACHED_ENTRY, "analyze_labels");
+
     // Labels in the document.
     for elem in document.introspector().all() {
         let Some(label) = elem.label() else { continue };

crates/tinymist-query/src/analysis.rs

@@ -1,7 +1,6 @@
 //! Semantic static and dynamic analysis of the source code.
 
 mod bib;
-
 pub(crate) use bib::*;
 pub mod call;
 pub use call::*;
@@ -15,30 +14,30 @@ pub mod doc_highlight;
 pub use doc_highlight::*;
 pub mod link_expr;
 pub use link_expr::*;
-pub mod stats;
-pub use stats::*;
 pub mod definition;
 pub use definition::*;
 pub mod signature;
 pub use signature::*;
 pub mod semantic_tokens;
 pub use semantic_tokens::*;
-use tinymist_std::error::WithContextUntyped;
-mod post_tyck;
-mod tyck;
-pub(crate) use crate::ty::*;
-pub(crate) use post_tyck::*;
-pub(crate) use tyck::*;
-mod prelude;
 
 mod global;
+mod post_tyck;
+mod prelude;
+mod tyck;
+
+pub(crate) use crate::ty::*;
 pub use global::*;
+pub(crate) use post_tyck::*;
+pub(crate) use tinymist_analysis::stats::{AnalysisStats, QueryStatGuard};
+pub(crate) use tyck::*;
 
 use std::sync::Arc;
 
 use ecow::eco_format;
 use lsp_types::Url;
 use tinymist_project::LspComputeGraph;
+use tinymist_std::error::WithContextUntyped;
 use tinymist_std::{Result, bail};
 use tinymist_world::{EntryReader, EntryState, TaskInputs};
 use typst::diag::{FileError, FileResult, StrResult};

crates/tinymist-query/src/analysis/global.rs

@@ -1115,13 +1115,7 @@ impl SharedContext {
     }
 
     fn query_stat(&self, id: TypstFileId, query: &'static str) -> QueryStatGuard {
-        let stats = &self.analysis.stats.query_stats;
-        let entry = stats.entry(id).or_default();
-        let entry = entry.entry(query).or_default();
-        QueryStatGuard {
-            bucket: entry.clone(),
-            since: tinymist_std::time::Instant::now(),
-        }
+        self.analysis.stats.stat(id, query)
     }
 
     /// Check on a module before really needing them. But we likely use them

crates/tinymist-query/src/analysis/stats.rs

@@ -1,115 +0,0 @@
-//! Statistics about the analyzers
-
-use std::sync::Arc;
-
-use parking_lot::Mutex;
-use tinymist_std::hash::FxDashMap;
-use tinymist_std::time::Duration;
-use typst::syntax::FileId;
-
-#[derive(Clone)]
-pub(crate) struct QueryStatBucketData {
-    pub query: u64,
-    pub missing: u64,
-    pub total: Duration,
-    pub min: Duration,
-    pub max: Duration,
-}
-
-impl Default for QueryStatBucketData {
-    fn default() -> Self {
-        Self {
-            query: 0,
-            missing: 0,
-            total: Duration::from_secs(0),
-            min: Duration::from_secs(u64::MAX),
-            max: Duration::from_secs(0),
-        }
-    }
-}
-
-/// Statistics about some query
-#[derive(Default, Clone)]
-pub(crate) struct QueryStatBucket {
-    pub data: Arc<Mutex<QueryStatBucketData>>,
-}
-
-pub(crate) struct QueryStatGuard {
-    pub bucket: QueryStatBucket,
-    pub since: tinymist_std::time::Instant,
-}
-
-impl Drop for QueryStatGuard {
-    fn drop(&mut self) {
-        let elapsed = self.since.elapsed();
-        let mut data = self.bucket.data.lock();
-        data.query += 1;
-        data.total += elapsed;
-        data.min = data.min.min(elapsed);
-        data.max = data.max.max(elapsed);
-    }
-}
-
-impl QueryStatGuard {
-    pub(crate) fn miss(&self) {
-        let mut data = self.bucket.data.lock();
-        data.missing += 1;
-    }
-}
-
-/// Statistics about the analyzers
-#[derive(Default)]
-pub struct AnalysisStats {
-    pub(crate) query_stats: FxDashMap<FileId, FxDashMap<&'static str, QueryStatBucket>>,
-}
-
-impl AnalysisStats {
-    /// Report the statistics of the analysis.
-    pub fn report(&self) -> String {
-        let stats = &self.query_stats;
-        let mut data = Vec::new();
-        for refs in stats.iter() {
-            let id = refs.key();
-            let queries = refs.value();
-            for refs2 in queries.iter() {
-                let query = refs2.key();
-                let bucket = refs2.value().data.lock().clone();
-                let name = format!("{id:?}:{query}").replace('\\', "/");
-                data.push((name, bucket));
-            }
-        }
-
-        // sort by query duration
-        data.sort_by(|x, y| y.1.max.cmp(&x.1.max));
-
-        // format to html
-
-        let mut html = String::new();
-        html.push_str(r#"<div>
-<style>
-table.analysis-stats { width: 100%; border-collapse: collapse; }
-table.analysis-stats th, table.analysis-stats td { border: 1px solid black; padding: 8px; text-align: center; }
-table.analysis-stats th.name-column, table.analysis-stats td.name-column { text-align: left; }
-table.analysis-stats tr:nth-child(odd) { background-color: rgba(242, 242, 242, 0.8); }
-@media (prefers-color-scheme: dark) {
-    table.analysis-stats tr:nth-child(odd) { background-color: rgba(50, 50, 50, 0.8); }
-}
-</style>
-<table class="analysis-stats"><tr><th class="query-column">Query</th><th>Count</th><th>Missing</th><th>Total</th><th>Min</th><th>Max</th></tr>"#);
-
-        for (name, bucket) in data {
-            html.push_str("<tr>");
-            html.push_str(&format!(r#"<td class="query-column">{name}</td>"#));
-            html.push_str(&format!("<td>{}</td>", bucket.query));
-            html.push_str(&format!("<td>{}</td>", bucket.missing));
-            html.push_str(&format!("<td>{:?}</td>", bucket.total));
-            html.push_str(&format!("<td>{:?}</td>", bucket.min));
-            html.push_str(&format!("<td>{:?}</td>", bucket.max));
-            html.push_str("</tr>");
-        }
-        html.push_str("</table>");
-        html.push_str("</div>");
-
-        html
-    }
-}

crates/tinymist-query/src/code_action.rs

@@ -127,6 +127,7 @@ mod tests {
         source: &Source,
         request_range: &LspRange,
     ) -> CodeActionContext {
+        // todo: reuse world compute graph APIs.
         let Warned {
             output,
             warnings: compiler_warnings,
@@ -144,8 +145,8 @@ mod tests {
         CodeActionContext {
             // The filtering here matches the LSP specification and VS Code behavior;
             // see https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#codeActionContext:
-            //   `diagnostics`: An array of diagnostics known on the client side overlapping the range
-            //   provided to the textDocument/codeAction request [...]
+            //   `diagnostics`: An array of diagnostics known on the client side overlapping the
+            // range   provided to the textDocument/codeAction request [...]
             diagnostics: diagnostics
                 .filter(|diag| ranges_overlap(&diag.range, request_range))
                 .collect(),

crates/tinymist-query/src/lib.rs

@@ -48,7 +48,7 @@ pub mod index;
 pub mod package;
 pub mod syntax;
 pub mod testing;
-pub use tinymist_analysis::{ty, upstream};
+pub use tinymist_analysis::{stats::GLOBAL_STATS, ty, upstream};
 
 /// The physical position in a document.
 pub type FramePosition = typst::layout::Position;

crates/tinymist-query/src/syntax/docs.rs

@@ -1,6 +1,7 @@
 use std::{collections::BTreeMap, ops::Deref, sync::LazyLock};
 
 use ecow::eco_format;
+use tinymist_analysis::stats::GLOBAL_STATS;
 use typst::foundations::{IntoValue, Module, Str, Type};
 
 use crate::{StrRef, adt::interner::Interned};
@@ -20,6 +21,8 @@ pub(crate) fn do_compute_docstring(
     docs: String,
     kind: DefKind,
 ) -> Option<DocString> {
+    let _guard = GLOBAL_STATS.stat(fid, "compute_docstring");
+
     let checker = DocsChecker {
         fid,
         ctx,

crates/tinymist-query/src/tests.rs

@@ -460,19 +460,6 @@ pub(crate) fn file_path_(uri: &lsp_types::Url) -> String {
     unix_slash(&rel_path)
 }
 
-pub struct HashRepr<T>(pub T);
-
-// sha256
-impl fmt::Display for HashRepr<JsonRepr> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use sha2::{Digest, Sha256};
-
-        let res = self.0.to_string();
-        let hash = Sha256::digest(res).to_vec();
-        write!(f, "sha256:{}", hex::encode(hash))
-    }
-}
-
 /// Extension methods for `Regex` that operate on `Cow<str>` instead of `&str`.
 pub trait RegexCowExt {
     /// [`Regex::replace_all`], but taking text as `Cow<str>` instead of `&str`.

crates/tinymist/src/project.rs

@@ -30,7 +30,7 @@ use sync_ls::{LspClient, TypedLspClient};
 use tinymist_project::vfs::{FileChangeSet, MemoryEvent};
 use tinymist_query::analysis::{Analysis, LspQuerySnapshot, PeriscopeProvider};
 use tinymist_query::{
-    CheckRequest, CompilerQueryRequest, DiagnosticsMap, LocalContext, SemanticRequest,
+    CheckRequest, CompilerQueryRequest, DiagnosticsMap, LocalContext, SemanticRequest, GLOBAL_STATS,
 };
 use tinymist_render::PeriscopeRenderer;
 use tinymist_std::{error::prelude::*, ImmutPath};
@@ -663,9 +663,15 @@ impl CompileHandler<LspCompilerFeat, ProjectInsStateExt> for CompileHandlerImpl
             let Some(compile_fn) = s.may_compile(&c.handler) else {
                 continue;
             };
+            let id = s
+                .snapshot()
+                .world()
+                .main_id()
+                .unwrap_or_else(|| *DETACHED_ENTRY);
 
             s.ext.compiling_since = Some(tinymist_std::time::now());
             spawn_cpu(move || {
+                let _guard = GLOBAL_STATS.stat(id, "main_compile");
                 compile_fn();
             });
         }

crates/tinymist/src/resource/symbols.rs

@@ -5,6 +5,7 @@ use reflexo_typst::TypstPagedDocument;
 use reflexo_typst::{vector::font::GlyphId, TypstFont};
 use reflexo_vec2svg::SvgGlyphBuilder;
 use sync_ls::LspResult;
+use tinymist_query::GLOBAL_STATS;
 use typst::foundations::Bytes;
 use typst::{syntax::VirtualPath, World};
 
@@ -230,6 +231,8 @@ fn render_symbols(
         entry: Some(new_entry),
         ..TaskInputs::default()
     });
+
+    let _guard = GLOBAL_STATS.stat(forked.main(), "render_symbols");
     forked
         .map_shadow_by_id(forked.main(), Bytes::from_string(math_shaping_text))
         .map_err(|e| error_once!("cannot map shadow", err: e))
@@ -360,7 +363,8 @@ fn create_display_svg(font: &TypstFont, gid: GlyphId, svg_path: &str) -> String
         .map(f32::from)
         .unwrap_or(units_per_em);
 
-    // Start viewBox.x at left-most ink or 0, whichever is smaller (to include left overhang)
+    // Start viewBox.x at left-most ink or 0, whichever is smaller (to include left
+    // overhang)
     let view_x = x_min.min(0.0);
 
     // Start view width as the advance; enlarge if ink extends past that

crates/tinymist/src/server.rs

@@ -8,7 +8,7 @@ use lsp_types::request::ShowMessageRequest;
 use lsp_types::*;
 use reflexo::debug_loc::LspPosition;
 use sync_ls::*;
-use tinymist_query::ServerInfoResponse;
+use tinymist_query::{ServerInfoResponse, GLOBAL_STATS};
 use tinymist_std::error::prelude::*;
 use tinymist_std::ImmutPath;
 use tokio::sync::mpsc;
@@ -497,6 +497,7 @@ impl ServerState {
         let dg = self.project.primary_id().to_string();
         let api_stats = self.project.stats.report();
         let query_stats = self.project.analysis.report_query_stats();
+        let global_stats = GLOBAL_STATS.report();
         let alloc_stats = self.project.analysis.report_alloc_stats();
 
         let snap = self.snapshot().map_err(internal_error)?;
@@ -509,6 +510,7 @@ impl ServerState {
                 inputs: w.inputs().as_ref().deref().clone(),
                 stats: HashMap::from_iter([
                     ("api".to_owned(), api_stats),
+                    ("global".to_owned(), global_stats),
                     ("query".to_owned(), query_stats),
                     ("alloc".to_owned(), alloc_stats),
                 ]),

crates/tinymist/src/task/export.rs

@@ -6,10 +6,10 @@ use std::sync::{Arc, OnceLock};
 use std::{ops::DerefMut, pin::Pin};
 
 use reflexo::ImmutPath;
-use reflexo_typst::{Bytes, CompilationTask, ExportComputation};
+use reflexo_typst::{Bytes, CompilationTask, ExportComputation, DETACHED_ENTRY};
 use sync_ls::{internal_error, just_future};
 use tinymist_project::LspWorld;
-use tinymist_query::{OnExportRequest, OnExportResponse, PagedExportResponse};
+use tinymist_query::{OnExportRequest, OnExportResponse, PagedExportResponse, GLOBAL_STATS};
 use tinymist_std::error::prelude::*;
 use tinymist_std::fs::paths::write_atomic;
 use tinymist_std::path::PathClean;
@@ -74,6 +74,9 @@ impl ServerState {
                 ..TaskInputs::default()
             });
 
+            let id = snap.world().main_id().unwrap_or_else(|| *DETACHED_ENTRY);
+            let _guard = GLOBAL_STATS.stat(id, "export");
+
             let is_html = matches!(task, ProjectTask::ExportHtml { .. });
             // todo: we may get some file missing errors here
             let artifact = CompiledArtifact::from_graph(snap.clone(), is_html);

tools/editor-tools/src/features/summary.ts

@@ -149,6 +149,55 @@ export const Summary = () => {
     return res;
   };
 
+  const fontStats = div(
+    { class: `card`, style: "flex: 1; width: 100%; padding: 10px" },
+    div(
+      { style: "position: relative; width: 100%; height: 0px" },
+      button(
+        {
+          class: "btn",
+          style: "position: absolute; top: 0px; right: 0px",
+          onclick: () => {
+            startModal(
+              div(
+                {
+                  style: "height: calc(100% - 20px); box-sizing: border-box; padding-top: 4px",
+                },
+                fontsExportPanel({
+                  fonts: docMetrics.val.fontInfo,
+                  sources: docMetrics.val.spanInfo.sources,
+                }),
+              ),
+            );
+          },
+        },
+        CopyIcon(),
+      ),
+    ),
+    div(van.derive(() => `This document uses ${docMetrics.val.fontInfo.length} fonts.`)),
+    (_dom?: Element) =>
+      div(
+        ...docMetrics.val.fontInfo
+          .sort((x, y) => {
+            if (x.usesScale === undefined || y.usesScale === undefined) {
+              if (x.usesScale === undefined) {
+                return 1;
+              }
+              if (y.usesScale === undefined) {
+                return -1;
+              }
+
+              return x.name.localeCompare(y.name);
+            }
+            if (x.usesScale !== y.usesScale) {
+              return y.usesScale - x.usesScale;
+            }
+            return x.name.localeCompare(y.name);
+          })
+          .map(FontSlot),
+      ),
+  );
+
   return div(
     {
       class: "flex-col",
@@ -159,54 +208,7 @@ export const Summary = () => {
       div(van.derive(() => `This document is compiled with following arguments.`)),
       div({ style: "margin: 1.2em; margin-left: 0.5em" }, ...ArgSlots()),
     ),
-    div(
-      { class: `card`, style: "flex: 1; width: 100%; padding: 10px" },
-      div(
-        { style: "position: relative; width: 100%; height: 0px" },
-        button(
-          {
-            class: "btn",
-            style: "position: absolute; top: 0px; right: 0px",
-            onclick: () => {
-              startModal(
-                div(
-                  {
-                    style: "height: calc(100% - 20px); box-sizing: border-box; padding-top: 4px",
-                  },
-                  fontsExportPanel({
-                    fonts: docMetrics.val.fontInfo,
-                    sources: docMetrics.val.spanInfo.sources,
-                  }),
-                ),
-              );
-            },
-          },
-          CopyIcon(),
-        ),
-      ),
-      div(van.derive(() => `This document uses ${docMetrics.val.fontInfo.length} fonts.`)),
-      (_dom?: Element) =>
-        div(
-          ...docMetrics.val.fontInfo
-            .sort((x, y) => {
-              if (x.usesScale === undefined || y.usesScale === undefined) {
-                if (x.usesScale === undefined) {
-                  return 1;
-                }
-                if (y.usesScale === undefined) {
-                  return -1;
-                }
-
-                return x.name.localeCompare(y.name);
-              }
-              if (x.usesScale !== y.usesScale) {
-                return y.usesScale - x.usesScale;
-              }
-              return x.name.localeCompare(y.name);
-            })
-            .map(FontSlot),
-        ),
-    ),
+    fontStats,
     div(
       {
         class: `card hidden`,