feat: initialize symbol picker (#155)

* feat: initialize symbol picker

* dev: update snapshot

* dev: remove mock

crates/tinymist/Cargo.toml

@@ -50,6 +50,7 @@ typst-ts-core = { workspace = true, default-features = false, features = [
     "vector-bbox",
     "no-content-hint",
 ] }
+typst-ts-svg-exporter.workspace = true
 codespan-reporting.workspace = true
 typst-ts-compiler.workspace = true
 toml.workspace = true

crates/tinymist/src/lib.rs

@@ -29,19 +29,20 @@
 // pub mod formatting;
 mod actor;
 pub mod harness;
-pub use crate::harness::LspHost;
+mod resource;
+mod server;
 mod state;
 mod task;
 mod tools;
 pub mod transport;
 mod utils;
 mod world;
-pub use world::{CompileFontOpts, CompileOnceOpts, CompileOpts, LspWorld, LspWorldBuilder};
-mod server;
+pub use crate::harness::LspHost;
 pub use server::compiler;
 pub use server::compiler_init;
 pub use server::lsp::*;
 pub use server::lsp_init::*;
+pub use world::{CompileFontOpts, CompileOnceOpts, CompileOpts, LspWorld, LspWorldBuilder};
 
 use lsp_server::ResponseError;
 

crates/tinymist/src/resource/mod.rs

@@ -0,0 +1,20 @@
+mod symbols;
+
+mod prelude {
+
+    pub use std::collections::HashMap;
+
+    pub use once_cell::sync::Lazy;
+    pub use serde::{Deserialize, Serialize};
+    pub use serde_json::Value as JsonValue;
+    pub use typst::foundations::{Scope, Value};
+    pub use typst::symbols::Symbol;
+    pub use typst_ts_compiler::service::Compiler;
+    pub use typst_ts_core::error::prelude::*;
+    pub use typst_ts_svg_exporter::ir::{GlyphItem, GlyphRef};
+    pub use typst_ts_svg_exporter::{DefaultExportFeature, SvgTask, SvgText};
+
+    pub use crate::TypstLanguageServer;
+
+    pub type Svg<'a> = SvgTask<'a, DefaultExportFeature>;
+}

crates/tinymist/src/resource/symbols.rs

@@ -0,0 +1,339 @@
+pub use super::prelude::*;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct ResourceSymbolResponse {
+    symbols: HashMap<String, ResourceSymbolItem>,
+    #[serde(rename = "fontSelects")]
+    font_selects: Vec<FontItem>,
+    #[serde(rename = "glyphDefs")]
+    glyph_defs: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct ResourceSymbolItem {
+    category: SymCategory,
+    unicode: u32,
+    glyphs: Vec<ResourceGlyphDesc>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+enum SymCategory {
+    #[serde(rename = "accent")]
+    Accent,
+    #[serde(rename = "greek")]
+    Greek,
+    #[serde(rename = "misc")]
+    Misc,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct ResourceGlyphDesc {
+    #[serde(rename = "fontIndex")]
+    font_index: u32,
+    #[serde(rename = "xAdvance")]
+    x_advance: Option<u16>,
+    #[serde(rename = "yAdvance")]
+    y_advance: Option<u16>,
+    #[serde(rename = "xMin")]
+    x_min: Option<i16>,
+    #[serde(rename = "xMax")]
+    x_max: Option<i16>,
+    #[serde(rename = "yMin")]
+    y_min: Option<i16>,
+    #[serde(rename = "yMax")]
+    y_max: Option<i16>,
+    name: Option<String>,
+    shape: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct FontItem {
+    family: String,
+    #[serde(rename = "capHeight")]
+    cap_height: f32,
+    ascender: f32,
+    descender: f32,
+    #[serde(rename = "unitsPerEm")]
+    units_per_em: f32,
+    // vertical: bool,
+}
+
+type ResourceSymbolMap = HashMap<String, ResourceSymbolItem>;
+
+static CAT_MAP: Lazy<HashMap<&str, SymCategory>> = Lazy::new(|| {
+    use SymCategory::*;
+
+    HashMap::from_iter([
+        ("sym.cancel", Accent),
+        ("sym.grave", Accent),
+        ("sym.acute", Accent),
+        ("sym.hat", Accent),
+        ("sym.widehat", Accent),
+        ("sym.tilde", Accent),
+        ("sym.macron", Accent),
+        ("sym.breve", Accent),
+        ("sym.dot", Accent),
+        ("sym.dot.double", Accent),
+        ("sym.dot.triple", Accent),
+        ("sym.dot.quad", Accent),
+        ("sym.acute.double", Accent),
+        ("sym.caron", Accent),
+        ("sym.breve", Accent),
+        ("sym.caron", Accent),
+        ("sym.circle", Accent),
+        ("sym.alpha", Greek),
+        ("sym.beta", Greek),
+        ("sym.gamma", Greek),
+        ("sym.delta", Greek),
+        ("sym.epsilon.alt", Greek),
+        ("sym.zeta", Greek),
+        ("sym.eta", Greek),
+        ("sym.theta", Greek),
+        ("sym.iota", Greek),
+        ("sym.kappa", Greek),
+        ("sym.lambda", Greek),
+        ("sym.mu", Greek),
+        ("sym.nu", Greek),
+        ("sym.xi", Greek),
+        ("sym.omicron", Greek),
+        ("sym.pi", Greek),
+        ("sym.rho", Greek),
+        ("sym.sigma", Greek),
+        ("sym.tau", Greek),
+        ("sym.upsilon", Greek),
+        ("sym.phi.alt", Greek),
+        ("sym.chi", Greek),
+        ("sym.psi", Greek),
+        ("sym.omega", Greek),
+        ("sym.Alpha", Greek),
+        ("sym.Beta", Greek),
+        ("sym.Gamma", Greek),
+        ("sym.Delta", Greek),
+        ("sym.Epsilon", Greek),
+        ("sym.Zeta", Greek),
+        ("sym.Eta", Greek),
+        ("sym.Theta", Greek),
+        ("sym.Iota", Greek),
+        ("sym.Kappa", Greek),
+        ("sym.Lambda", Greek),
+        ("sym.Mu", Greek),
+        ("sym.Nu", Greek),
+        ("sym.Xi", Greek),
+        ("sym.Omicron", Greek),
+        ("sym.Pi", Greek),
+        ("sym.Rho", Greek),
+        ("sym.Sigma", Greek),
+        ("sym.Tau", Greek),
+        ("sym.Upsilon", Greek),
+        ("sym.Phi", Greek),
+        ("sym.Chi", Greek),
+        ("sym.Psi", Greek),
+        ("sym.Omega", Greek),
+        ("sym.beta.alt", Greek),
+        ("sym.epsilon", Greek),
+        ("sym.kappa.alt", Greek),
+        ("sym.phi", Greek),
+        ("sym.pi.alt", Greek),
+        ("sym.rho.alt", Greek),
+        ("sym.sigma.alt", Greek),
+        ("sym.theta.alt", Greek),
+        ("sym.ell", Greek),
+    ])
+});
+
+impl TypstLanguageServer {
+    /// Get the all valid symbols
+    pub fn get_symbol_resources(&self) -> ZResult<JsonValue> {
+        let mut symbols = ResourceSymbolMap::new();
+        populate_scope(typst::symbols::sym().scope(), "sym", &mut symbols);
+        // currently we don't have plan on emoji
+        // populate_scope(typst::symbols::emoji().scope(), "emoji", &mut symbols);
+
+        let chars = symbols
+            .values()
+            .map(|e| char::from_u32(e.unicode).unwrap())
+            .collect::<String>();
+
+        let font = self
+            .primary()
+            .steal(move |e| {
+                use typst::text::FontVariant;
+                use typst::World;
+                let book = e.compiler.world().book();
+
+                // todo: bad font fallback
+
+                let fonts = &["New Computer Modern Math", "Latin Modern Math", "Cambria"];
+
+                let preferred_fonts = fonts
+                    .iter()
+                    .flat_map(|f| {
+                        let f = f.to_lowercase();
+                        book.select(&f, FontVariant::default())
+                            .and_then(|i| e.compiler.world().font(i))
+                    })
+                    .collect::<Vec<_>>();
+
+                let mut last_hit: Option<typst_ts_core::TypstFont> = None;
+
+                log::info!("font init: {hit:?}", hit = last_hit);
+
+                let fonts = chars
+                    .chars()
+                    .map(|c| {
+                        for font in &preferred_fonts {
+                            if font.info().coverage.contains(c as u32) {
+                                return Some(font.clone());
+                            }
+                        }
+
+                        if let Some(last_hit) = &last_hit {
+                            if last_hit.info().coverage.contains(c as u32) {
+                                return Some(last_hit.clone());
+                            }
+                        }
+
+                        let hit =
+                            book.select_fallback(None, FontVariant::default(), &c.to_string())?;
+                        last_hit = e.compiler.world().font(hit);
+
+                        log::info!("font hit: {hit:?}", hit = last_hit);
+
+                        last_hit.clone()
+                    })
+                    .collect::<Vec<_>>();
+
+                fonts
+            })
+            .ok();
+
+        let mut glyph_def = String::new();
+
+        let mut collected_fonts = None;
+
+        if let Some(fonts) = font.clone() {
+            let glyph_provider = typst_ts_core::font::GlyphProvider::default();
+            let glyph_pass =
+                typst_ts_core::vector::pass::ConvertInnerImpl::new(glyph_provider, false);
+
+            let mut glyph_renderer = Svg::default();
+            let mut glyphs = vec![];
+
+            let font_collected = fonts
+                .iter()
+                .flatten()
+                .collect::<std::collections::HashSet<_>>()
+                .into_iter()
+                .cloned()
+                .collect::<Vec<_>>();
+
+            let mut render_char = |font: Option<&typst_ts_core::TypstFont>, u| {
+                let font = font?;
+                let font_index = font_collected.iter().position(|e| e == font).unwrap() as u32;
+
+                let id = font.ttf().glyph_index(char::from_u32(u).unwrap())?;
+                let width = font.ttf().glyph_hor_advance(id);
+                let height = font.ttf().glyph_ver_advance(id);
+                let bbox = font.ttf().glyph_bounding_box(id);
+
+                let glyph = glyph_pass.must_flat_glyph(&GlyphItem::Raw(font.clone(), id))?;
+
+                let g_ref = GlyphRef {
+                    font_hash: font_index,
+                    glyph_idx: id.0 as u32,
+                };
+
+                glyphs.push((g_ref, glyph));
+
+                Some(ResourceGlyphDesc {
+                    font_index,
+                    x_advance: width,
+                    y_advance: height,
+                    x_min: bbox.map(|e| e.x_min),
+                    x_max: bbox.map(|e| e.x_max),
+                    y_min: bbox.map(|e| e.y_min),
+                    y_max: bbox.map(|e| e.y_max),
+                    name: font.ttf().glyph_name(id).map(|e| e.to_owned()),
+                    shape: Some(g_ref.as_svg_id("g")),
+                })
+            };
+
+            for ((_, v), font) in symbols.iter_mut().zip(fonts.iter()) {
+                let Some(desc) = render_char(font.as_ref(), v.unicode) else {
+                    continue;
+                };
+
+                v.glyphs.push(desc);
+            }
+
+            let mut svg = vec![];
+
+            // attach the glyph defs
+            svg.push(r#"<defs class="glyph">"#.into());
+            svg.extend(glyph_renderer.render_glyphs(glyphs.iter().map(|(id, item)| (*id, item))));
+            svg.push("</defs>".into());
+
+            glyph_def = SvgText::join(svg);
+
+            collected_fonts = Some(font_collected);
+        }
+
+        let resp = ResourceSymbolResponse {
+            symbols,
+            font_selects: collected_fonts
+                .map(|e| e.into_iter())
+                .into_iter()
+                .flatten()
+                .map(|e| FontItem {
+                    family: e.info().family.clone(),
+                    cap_height: e.metrics().cap_height.get() as f32,
+                    ascender: e.metrics().ascender.get() as f32,
+                    descender: e.metrics().descender.get() as f32,
+                    units_per_em: e.metrics().units_per_em as f32,
+                })
+                .collect::<Vec<_>>(),
+            glyph_defs: glyph_def,
+        };
+
+        serde_json::to_value(resp).context("cannot serialize response")
+    }
+}
+
+fn populate(sym: &Symbol, mod_name: &str, sym_name: &str, out: &mut ResourceSymbolMap) {
+    for (modifier_name, ch) in sym.variants() {
+        let mut name =
+            String::with_capacity(mod_name.len() + sym_name.len() + modifier_name.len() + 2);
+
+        name.push_str(mod_name);
+        name.push('.');
+        name.push_str(sym_name);
+
+        if !modifier_name.is_empty() {
+            name.push('.');
+            name.push_str(modifier_name);
+        }
+
+        let category = CAT_MAP
+            .get(name.as_str())
+            .cloned()
+            .unwrap_or(SymCategory::Misc);
+        out.insert(
+            name,
+            ResourceSymbolItem {
+                category,
+                unicode: ch as u32,
+                glyphs: vec![],
+            },
+        );
+    }
+}
+
+fn populate_scope(sym: &Scope, mod_name: &str, out: &mut ResourceSymbolMap) {
+    for (k, v) in sym.iter() {
+        let Value::Symbol(sym) = v else {
+            continue;
+        };
+
+        populate(sym, mod_name, k, out)
+    }
+}

crates/tinymist/src/server/lsp.rs

@@ -64,13 +64,14 @@ type LspHandler<Req, Res> = fn(srv: &mut TypstLanguageServer, args: Req) -> LspR
 
 /// Returns Ok(Some()) -> Already responded
 /// Returns Ok(None) -> Need to respond none
-/// Returns Err(..) -> Need t o respond error
+/// Returns Err(..) -> Need to respond error
 type LspRawHandler =
     fn(srv: &mut TypstLanguageServer, args: (RequestId, JsonValue)) -> LspResult<Option<()>>;
 
 type ExecuteCmdMap = HashMap<&'static str, LspHandler<Vec<JsonValue>, JsonValue>>;
 type NotifyCmdMap = HashMap<&'static str, LspMethod<()>>;
 type RegularCmdMap = HashMap<&'static str, LspRawHandler>;
+type ResourceMap = HashMap<ImmutPath, LspHandler<Vec<JsonValue>, JsonValue>>;
 
 macro_rules! exec_fn {
     ($ty: ty, Self::$method: ident, $($arg_key:ident),+ $(,)?) => {{
@@ -186,6 +187,8 @@ pub struct TypstLanguageServer {
     pub notify_cmds: NotifyCmdMap,
     /// Regular commands for dispatching.
     pub regular_cmds: RegularCmdMap,
+    /// Regular commands for dispatching.
+    pub resources_routes: ResourceMap,
 
     // Resources
     /// The semantic token context.
@@ -230,6 +233,7 @@ impl TypstLanguageServer {
             exec_cmds: Self::get_exec_commands(),
             regular_cmds: Self::get_regular_cmds(),
             notify_cmds: Self::get_notify_cmds(),
+            resources_routes: Self::get_resources_routes(),
 
             pinning: false,
             focusing: None,
@@ -598,6 +602,8 @@ impl TypstLanguageServer {
             redirected_command!("tinymist.doInitTemplate", Self::init_template),
             redirected_command!("tinymist.doGetTemplateEntry", Self::do_get_template_entry),
             redirected_command!("tinymist.getDocumentMetrics", Self::get_document_metrics),
+            // For Documentations
+            redirected_command!("tinymist.getResources", Self::get_resources),
         ])
     }
 
@@ -785,6 +791,48 @@ impl TypstLanguageServer {
     }
 }
 
+impl TypstLanguageServer {
+    fn get_resources_routes() -> ResourceMap {
+        macro_rules! resources_at {
+            ($key: expr, Self::$method: ident) => {
+                (
+                    Path::new($key).clean().as_path().into(),
+                    exec_fn!(LspHandler<Vec<JsonValue>, JsonValue>, Self::$method, inputs),
+                )
+            };
+        }
+
+        ResourceMap::from_iter([
+            resources_at!("/symbols", Self::resources_alt_symbols),
+            resources_at!("/tutorial", Self::resource_tutoral),
+        ])
+    }
+
+    /// Get static resources with help of tinymist service, for example, a
+    /// static help pages for some typst function.
+    pub fn get_resources(&mut self, arguments: Vec<JsonValue>) -> LspResult<JsonValue> {
+        let u = parse_path(arguments.first())?;
+
+        let Some(handler) = self.resources_routes.get(u.as_ref()) else {
+            error!("asked for unknown resource: {u:?}");
+            return Err(method_not_found());
+        };
+
+        // Note our redirection will keep the first path argument in the arguments vec.
+        handler(self, arguments)
+    }
+    /// Get the all valid symbols
+    pub fn resources_alt_symbols(&self, _arguments: Vec<JsonValue>) -> LspResult<JsonValue> {
+        let resp = self.get_symbol_resources();
+        resp.map_err(|e| internal_error(e.to_string()))
+    }
+
+    /// Get tutorial web page
+    pub fn resource_tutoral(&self, _arguments: Vec<JsonValue>) -> LspResult<JsonValue> {
+        Err(method_not_found())
+    }
+}
+
 /// Document Synchronization
 impl TypstLanguageServer {
     fn did_open(&mut self, params: DidOpenTextDocumentParams) -> LspResult<()> {
@@ -1049,11 +1097,7 @@ fn parse_opts(v: Option<&JsonValue>) -> LspResult<ExportOpts> {
 fn parse_path(v: Option<&JsonValue>) -> LspResult<ImmutPath> {
     let new_entry = match v {
         Some(JsonValue::String(s)) => Path::new(s).clean().as_path().into(),
-        _ => {
-            return Err(invalid_params(
-                "The first parameter is not a valid path or null",
-            ))
-        }
+        _ => return Err(invalid_params("The first parameter is not a valid path")),
     };
 
     Ok(new_entry)