feat: add js package registry support for tinymist-wasm (#2102)

Co-authored-by: Myriad-Dreamin <camiyoru@gmail.com>

Cargo.lock

@@ -4162,6 +4162,7 @@ dependencies = [
  "reflexo-vec2svg",
  "rpds",
  "serde",
+ "serde-wasm-bindgen",
  "serde_json",
  "serde_yaml",
  "strum",
@@ -4195,6 +4196,7 @@ dependencies = [
  "vergen",
  "walkdir",
  "wasm-bindgen",
+ "wasm-bindgen-futures",
 ]
 
 [[package]]
@@ -4454,6 +4456,7 @@ dependencies = [
  "comemo",
  "dirs",
  "ecow",
+ "js-sys",
  "log",
  "notify",
  "parking_lot",
@@ -4471,6 +4474,7 @@ dependencies = [
  "toml",
  "typst",
  "typst-assets",
+ "wasm-bindgen",
 ]
 
 [[package]]

crates/sync-lsp/src/server.rs

@@ -207,14 +207,18 @@ impl LspClientRoot {
 
     /// Creates a new language server host from js.
     #[cfg(feature = "web")]
-    pub fn new_js(handle: tokio::runtime::Handle, transport: JsTransportSender) -> Self {
+    pub fn new_js(handle: tokio::runtime::Handle, sender: JsTransportSender) -> Self {
         let dummy = dummy_transport::<LspMessage>();
 
         let _strong = Arc::new(dummy.sender.into());
         let weak = LspClient {
             handle,
             msg_kind: LspMessage::MESSAGE_KIND,
-            sender: TransportHost::Js(transport),
+            sender: TransportHost::Js {
+                event_id: Arc::new(AtomicU32::new(0)),
+                events: Arc::new(Mutex::new(HashMap::new())),
+                sender,
+            },
             req_queue: Arc::new(Mutex::new(ReqQueue::default())),
 
             hook: Arc::new(()),
@@ -237,45 +241,44 @@ impl LspClientRoot {
 type ReqHandler = Box<dyn for<'a> FnOnce(&'a mut dyn Any, LspOrDapResponse) + Send + Sync>;
 type ReqQueue = req_queue::ReqQueue<(String, Instant), ReqHandler>;
 
+/// Different transport mechanisms for communication.
 #[derive(Debug, Clone)]
-enum TransportHost {
+pub enum TransportHost {
+    /// System-level transport using native OS capabilities.
     System(SystemTransportSender),
+    /// JavaScript/WebAssembly transport for web environments.
     #[cfg(feature = "web")]
-    Js(JsTransportSender),
+    Js {
+        /// Atomic counter for generating unique event identifiers.
+        event_id: Arc<AtomicU32>,
+        /// Thread-safe storage for pending events indexed by their IDs.
+        events: Arc<Mutex<HashMap<u32, Event>>>,
+        /// The actual sender implementation for JavaScript environments.
+        sender: JsTransportSender,
+    },
 }
 
+/// A sender implementation for system-level transport operations.
 #[derive(Debug, Clone)]
-struct SystemTransportSender {
+pub struct SystemTransportSender {
+    /// Weak reference to the connection transmitter.
     pub(crate) sender: Weak<ConnectionTx>,
 }
 
 /// Creates a new js transport host.
 #[cfg(feature = "web")]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
 pub struct JsTransportSender {
-    event_id: Arc<AtomicU32>,
+    #[serde(with = "serde_wasm_bindgen::preserve")]
-    events: Arc<Mutex<HashMap<u32, Event>>>,
+    pub(crate) send_event: js_sys::Function,
-    pub(crate) sender_event: js_sys::Function,
+    #[serde(with = "serde_wasm_bindgen::preserve")]
-    pub(crate) sender_request: js_sys::Function,
+    pub(crate) send_request: js_sys::Function,
-    pub(crate) sender_notification: js_sys::Function,
+    #[serde(with = "serde_wasm_bindgen::preserve")]
-}
+    pub(crate) send_notification: js_sys::Function,
-
+    /// The acutal resolving function in JavaScript
-#[cfg(feature = "web")]
+    #[serde(with = "serde_wasm_bindgen::preserve")]
-impl JsTransportSender {
+    pub resolve_fn: js_sys::Function,
-    /// Creates a new JS transport host.
-    pub fn new(
-        sender_event: js_sys::Function,
-        sender_request: js_sys::Function,
-        sender_notification: js_sys::Function,
-    ) -> Self {
-        Self {
-            event_id: Arc::new(AtomicU32::new(0)),
-            events: Arc::new(Mutex::new(HashMap::new())),
-            sender_event,
-            sender_request,
-            sender_notification,
-        }
-    }
 }
 
 #[cfg(feature = "web")]
@@ -303,17 +306,19 @@ impl TransportHost {
                 }
             }
             #[cfg(feature = "web")]
-            TransportHost::Js(host) => {
+            TransportHost::Js {
+                event_id,
+                sender,
+                events,
+            } => {
                 let event_id = {
-                    let event_id = host
+                    let event_id = event_id.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                        .event_id
+                    let mut lg = events.lock();
-                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
-                    let mut lg = host.events.lock();
                     lg.insert(event_id, Box::new(event));
                     js_sys::Number::from(event_id)
                 };
-                if let Err(err) = host
+                if let Err(err) = sender
-                    .sender_event
+                    .send_event
                     .call1(&wasm_bindgen::JsValue::UNDEFINED, &event_id.into())
                 {
                     log::error!("failed to send event: {err:?}");
@@ -322,6 +327,7 @@ impl TransportHost {
         }
     }
 
+    /// Sends a message.
     pub fn send_message(&self, response: Message) {
         match self {
             TransportHost::System(host) => {
@@ -334,12 +340,12 @@ impl TransportHost {
                 }
             }
             #[cfg(feature = "web")]
-            TransportHost::Js(host) => match response {
+            TransportHost::Js { sender, .. } => match response {
                 #[cfg(feature = "lsp")]
                 Message::Lsp(lsp::Message::Request(req)) => {
                     let msg = to_js_value(&req).expect("failed to serialize request to js value");
-                    if let Err(err) = host
+                    if let Err(err) = sender
-                        .sender_request
+                        .send_request
                         .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg)
                     {
                         log::error!("failed to send request: {err:?}");
@@ -348,8 +354,8 @@ impl TransportHost {
                 #[cfg(feature = "lsp")]
                 Message::Lsp(lsp::Message::Notification(req)) => {
                     let msg = to_js_value(&req).expect("failed to serialize request to js value");
-                    if let Err(err) = host
+                    if let Err(err) = sender
-                        .sender_notification
+                        .send_notification
                         .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg)
                     {
                         log::error!("failed to send request: {err:?}");
@@ -362,8 +368,8 @@ impl TransportHost {
                 #[cfg(feature = "dap")]
                 Message::Dap(dap::Message::Request(req)) => {
                     let msg = to_js_value(&req).expect("failed to serialize request to js value");
-                    if let Err(err) = host
+                    if let Err(err) = sender
-                        .sender_request
+                        .send_request
                         .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg)
                     {
                         log::error!("failed to send request: {err:?}");
@@ -372,8 +378,8 @@ impl TransportHost {
                 #[cfg(feature = "dap")]
                 Message::Dap(dap::Message::Event(req)) => {
                     let msg = to_js_value(&req).expect("failed to serialize request to js value");
-                    if let Err(err) = host
+                    if let Err(err) = sender
-                        .sender_notification
+                        .send_notification
                         .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg)
                     {
                         log::error!("failed to send request: {err:?}");
@@ -404,7 +410,8 @@ pub struct LspClient {
     pub handle: tokio::runtime::Handle,
 
     pub(crate) msg_kind: MessageKind,
-    sender: TransportHost,
+    /// The TransportHost between LspClient and LspServer
+    pub sender: TransportHost,
     pub(crate) req_queue: Arc<Mutex<ReqQueue>>,
 
     pub(crate) hook: Arc<dyn LsHook>,

crates/sync-lsp/src/server/lsp_srv.rs

@@ -260,7 +260,7 @@ where
     #[cfg(feature = "web")]
     pub fn on_server_event(&mut self, event_id: u32) {
         let evt = match &self.client.sender {
-            TransportHost::Js(sender) => sender.events.lock().remove(&event_id),
+            TransportHost::Js { events, .. } => events.lock().remove(&event_id),
             TransportHost::System(_) => {
                 panic!("cannot send server event in system transport");
             }
@@ -403,4 +403,15 @@ where
             }
         }
     }
+
+    /// Handles an incoming response.
+    pub fn on_lsp_response(&mut self, resp: lsp::Response) {
+        let client = self.client.clone();
+        let Some(s) = self.state_mut() else {
+            log::warn!("server is not ready yet, while received response");
+            return;
+        };
+
+        client.complete_lsp_request(s, resp)
+    }
 }

crates/tinymist-package/src/registry/browser.rs

@@ -3,6 +3,7 @@
 use std::{io::Read, path::Path};
 
 use js_sys::Uint8Array;
+use tinymist_std::ImmutPath;
 use typst::diag::{EcoString, eco_format};
 use wasm_bindgen::{JsValue, prelude::*};
 
@@ -120,6 +121,19 @@ impl PackageRegistry for JsRegistry {
     }
 }
 
+impl JsRegistry {
+    /// Returns the path at which non-local packages should be stored when
+    /// downloaded.
+    pub fn package_cache_path(&self) -> Option<&ImmutPath> {
+        None
+    }
+
+    /// Returns the path at which local packages are stored.
+    pub fn package_path(&self) -> Option<&ImmutPath> {
+        None
+    }
+}
+
 // todo
 /// Safety: `JsRegistry` is only used in the browser environment, and we cannot
 /// share data between workers.

crates/tinymist-project/Cargo.toml

@@ -34,7 +34,9 @@ tinymist-l10n.workspace = true
 toml = { workspace = true, optional = true }
 typst.workspace = true
 typst-assets.workspace = true
-notify.workspace = true
+notify = { workspace = true, optional = true }
+js-sys = { workspace = true, optional = true }
+wasm-bindgen = { workspace = true, optional = true }
 
 [features]
 
@@ -43,8 +45,8 @@ no-content-hint = ["tinymist-task/no-content-hint"]
 lsp = ["toml"]
 # "system", 
 
-system = ["tinymist-std/system", "tinymist-world/system", "toml"]
+system = ["tinymist-std/system", "tinymist-world/system", "toml", "notify"]
-web = ["tinymist-std/web", "tinymist-world/web"]
+web = ["tinymist-std/web", "tinymist-world/web", "js-sys", "wasm-bindgen"]
 
 [lints]
 workspace = true

crates/tinymist-project/src/lsp.rs

@@ -5,6 +5,8 @@ use tinymist_std::ImmutPath;
 use tinymist_std::error::prelude::*;
 use tinymist_task::ExportTarget;
 use tinymist_world::package::RegistryPathMapper;
+#[cfg(all(not(feature = "system"), feature = "web"))]
+use tinymist_world::package::registry::ProxyContext;
 use tinymist_world::vfs::Vfs;
 use tinymist_world::{
     CompileSnapshot, CompilerFeat, CompilerUniverse, CompilerWorld, EntryOpts, EntryState,
@@ -28,12 +30,11 @@ impl CompilerFeat for LspCompilerFeat {
     type FontResolver = FontResolverImpl;
     /// It accesses a physical file system.
     type AccessModel = DynAccessModel;
-    /// It performs native HTTP requests for fetching package data.
+    /// It performs:
-    #[cfg(feature = "system")]
+    /// - native HTTP requests for fetching package data in system environment
-    type Registry = tinymist_world::package::registry::HttpRegistry;
+    /// - js proxied requests to browser environment
-    // todo: registry in browser
+    /// - no package registry in other environments
-    #[cfg(not(feature = "system"))]
+    type Registry = LspRegistry;
-    type Registry = tinymist_world::package::registry::DummyRegistry;
 }
 
 /// LSP universe that spawns LSP worlds.
@@ -211,10 +212,12 @@ impl WorldProvider for (crate::ProjectInput, ImmutPath) {
     }
 }
 
-#[cfg(not(feature = "system"))]
+#[cfg(all(not(feature = "system"), feature = "web"))]
-type LspRegistry = tinymist_world::package::registry::DummyRegistry;
+type LspRegistry = tinymist_world::package::registry::JsRegistry;
 #[cfg(feature = "system")]
 type LspRegistry = tinymist_world::package::registry::HttpRegistry;
+#[cfg(not(any(feature = "system", feature = "web")))]
+type LspRegistry = tinymist_world::package::registry::DummyRegistry;
 
 /// Builder for LSP universe.
 pub struct LspUniverseBuilder;
@@ -318,7 +321,20 @@ impl LspUniverseBuilder {
     }
 
     /// Resolves package registry from given options.
-    #[cfg(not(feature = "system"))]
+    #[cfg(all(not(feature = "system"), feature = "web"))]
+    pub fn resolve_package(
+        _cert_path: Option<ImmutPath>,
+        _args: Option<&CompilePackageArgs>,
+        resolve_fn: js_sys::Function,
+    ) -> tinymist_world::package::registry::JsRegistry {
+        tinymist_world::package::registry::JsRegistry {
+            context: ProxyContext::new(wasm_bindgen::JsValue::NULL),
+            real_resolve_fn: resolve_fn,
+        }
+    }
+
+    /// Resolves package registry from given options.
+    #[cfg(not(any(feature = "system", feature = "web")))]
     pub fn resolve_package(
         _cert_path: Option<ImmutPath>,
         _args: Option<&CompilePackageArgs>,

crates/tinymist/Cargo.toml

@@ -90,6 +90,8 @@ hyper-tungstenite = { workspace = true, optional = true }
 console_error_panic_hook = { version = "0.1.2", optional = true }
 js-sys = { version = "0.3.77", optional = true }
 wasm-bindgen = { version = "0.2.100", optional = true }
+serde-wasm-bindgen = { version = "0.6.5", optional = true }
+wasm-bindgen-futures = { version = "0.4.50", optional = true }
 
 [dev-dependencies]
 temp-env.workspace = true
@@ -119,8 +121,11 @@ web = [
     "console_error_panic_hook",
     "js-sys",
     "wasm-bindgen",
+    "serde-wasm-bindgen",
+    "wasm-bindgen-futures",
     "reflexo-typst/web",
     "tinymist-project/web",
+    "sync-ls/web",
 ]
 open = ["dep:open"]
 system = [

crates/tinymist/src/project.rs

@@ -96,6 +96,14 @@ impl ServerState {
             self.dep_tx.clone(),
             #[cfg(feature = "preview")]
             self.preview.watchers.clone(),
+            #[cfg(all(not(feature = "system"), feature = "web"))]
+            if let sync_ls::TransportHost::Js { sender, .. } =
+                self.client.clone().to_untyped().sender
+            {
+                sender.resolve_fn
+            } else {
+                panic!("Expected Js TransportHost")
+            },
         );
 
         let mut old_project = std::mem::replace(&mut self.project, new_project);
@@ -139,6 +147,7 @@ impl ServerState {
         client: TypedLspClient<ServerState>,
         dep_tx: mpsc::UnboundedSender<NotifyMessage>,
         #[cfg(feature = "preview")] preview: ProjectPreviewState,
+        #[cfg(all(not(feature = "system"), feature = "web"))] resolve_fn: js_sys::Function,
     ) -> ProjectState {
         let const_config = &config.const_config;
 
@@ -199,7 +208,13 @@ impl ServerState {
         log::info!("ServerState: creating ProjectState, entry: {entry:?}, inputs: {inputs:?}");
 
         let fonts = config.fonts();
+
+        #[cfg(all(not(feature = "system"), feature = "web"))]
+        let packages =
+            LspUniverseBuilder::resolve_package(cert_path.clone(), Some(&package), resolve_fn);
+        #[cfg(any(feature = "system", not(feature = "web")))]
         let packages = LspUniverseBuilder::resolve_package(cert_path.clone(), Some(&package));
+
         let creation_timestamp = config.creation_timestamp();
         let verse = LspUniverseBuilder::build(
             entry,

crates/tinymist/src/server.rs

@@ -131,6 +131,7 @@ impl ServerState {
 
         #[cfg(feature = "preview")]
         let watchers = crate::project::ProjectPreviewState::default();
+
         let handle = Self::project(
             &config,
             editor_tx.clone(),
@@ -138,6 +139,12 @@ impl ServerState {
             dep_tx.clone(),
             #[cfg(feature = "preview")]
             watchers.clone(),
+            #[cfg(all(not(feature = "system"), feature = "web"))]
+            if let TransportHost::Js { sender, .. } = client.clone().to_untyped().sender {
+                sender.resolve_fn
+            } else {
+                panic!("Expected Js TransportHost")
+            },
         );
 
         Self {

crates/tinymist/src/web.rs

@@ -2,10 +2,18 @@
 
 #![allow(unused)]
 
+use std::sync::LazyLock;
+
+use futures::future::MaybeDone;
 use js_sys::{Function, Promise};
+use sync_ls::{
+    internal_error, invalid_params, JsTransportSender, LsDriver, LspBuilder, LspClientRoot,
+    LspMessage, ResponseError,
+};
+use tinymist_project::CompileFontArgs;
 use wasm_bindgen::prelude::*;
 
-use crate::LONG_VERSION;
+use crate::{RegularInit, ServerState, LONG_VERSION};
 
 /// Gets the long version description of the library.
 #[wasm_bindgen]
@@ -13,39 +21,149 @@ pub fn version() -> String {
     LONG_VERSION.clone()
 }
 
-/// The Tinymist Language Server for WebAssembly.
+/// TinymistLanguageServer implements the LSP protocol for Typst documents
+/// in a WebAssembly environment
 #[wasm_bindgen]
 pub struct TinymistLanguageServer {
-    send_diagnostics: Function,
+    /// The client root that strongly references the LSP client.
-    send_request: Function,
+    _client: LspClientRoot,
-    send_notification: Function,
+    /// The mutable state of the server.
+    state: LsDriver<LspMessage, RegularInit>,
 }
 
 #[wasm_bindgen]
 impl TinymistLanguageServer {
-    /// Creates a new instance of the Tinymist Language Server.
+    /// Creates a new language server.
     #[wasm_bindgen(constructor)]
-    pub fn new(
+    pub fn new(init_opts: JsValue) -> Result<Self, JsValue> {
-        send_diagnostics: Function,
+        let sender = serde_wasm_bindgen::from_value::<JsTransportSender>(init_opts)
-        send_request: Function,
+            .map_err(|err| JsValue::from_str(&format!("Failed to deserialize init opts: {err}")))?;
-        send_notification: Function,
+
-    ) -> Self {
         std::panic::set_hook(Box::new(console_error_panic_hook::hook));
 
-        Self {
+        let _client = LspClientRoot::new_js(RUNTIMES.tokio_runtime.handle().clone(), sender);
-            send_diagnostics,
+        // Starts logging
-            send_request,
+        let _ = crate::init_log(crate::InitLogOpts {
-            send_notification,
+            is_transient_cmd: false,
+            is_test_no_verbose: false,
+            output: Some(_client.weak()),
+        });
+        let state = ServerState::install_lsp(LspBuilder::new(
+            RegularInit {
+                client: _client.weak().to_typed(),
+                font_opts: CompileFontArgs::default(),
+                exec_cmds: Vec::new(),
+            },
+            _client.weak(),
+        ))
+        .build();
+
+        Ok(Self { _client, state })
     }
+
+    /// Handles internal events.
+    pub fn on_event(&mut self, event_id: u32) {
+        self.state.on_server_event(event_id);
     }
 
     /// Handles incoming requests.
-    pub fn on_request(&self, method: String, js_params: JsValue) -> Result<JsValue, JsValue> {
+    pub fn on_request(&mut self, method: String, js_params: JsValue) -> JsValue {
-        todo!()
+        let params = serde_wasm_bindgen::from_value::<serde_json::Value>(js_params);
+        let params = match params {
+            Ok(p) => p,
+            Err(err) => return lsp_err(invalid_params(err)),
+        };
+
+        let result = self.state.on_lsp_request(&method, params);
+
+        match result {
+            Ok(MaybeDone::Done(Ok(t))) => lsp_serialize(&t),
+            Ok(MaybeDone::Done(Err(err))) => lsp_err(err),
+            // tokio doesn't get scheduled anymore after returning to js world
+            Ok(MaybeDone::Future(fut)) => wasm_bindgen_futures::future_to_promise(async move {
+                Ok(match fut.await {
+                    Ok(t) => lsp_serialize(&t),
+                    Err(err) => lsp_err(err),
+                })
+            })
+            .into(),
+            // match futures::executor::block_on(fut) {
+            //     Ok(t) => lsp_serialize(&t),
+            //     Err(err) => lsp_err(err),
+            // },
+            Ok(MaybeDone::Gone) => lsp_err(internal_error("response was weirdly gone")),
+            Err(err) => lsp_err(err),
+        }
     }
 
     /// Handles incoming notifications.
-    pub fn on_notification(&self, method: String, js_params: JsValue) -> Promise {
+    pub fn on_notification(&mut self, method: String, js_params: JsValue) {
-        todo!()
+        let params = serde_wasm_bindgen::from_value::<serde_json::Value>(js_params);
+        let params = match params {
+            Ok(p) => p,
+            Err(err) => {
+                log::error!("Failed to deserialize notification params: {err}");
+                return;
+            }
+        };
+
+        let err = self.state.on_notification(&method, params);
+        if let Err(err) = err {
+            log::error!("Failed to handle notification {method}: {err:?}");
         }
     }
+
+    /// Handles incoming responses.
+    pub fn on_response(&mut self, js_result: JsValue) {
+        let result = serde_wasm_bindgen::from_value::<sync_ls::lsp::Response>(js_result);
+        let resp = match result {
+            Ok(r) => r,
+            Err(err) => {
+                log::error!("Failed to deserialize response: {err}");
+                return;
+            }
+        };
+        self.state.on_lsp_response(resp);
+    }
+
+    /// Get the version of the language server.
+    pub fn version() -> String {
+        env!("CARGO_PKG_VERSION").to_string()
+    }
+}
+
+/// The runtimes used by the application.
+pub struct Runtimes {
+    /// The tokio runtime.
+    pub tokio_runtime: tokio::runtime::Runtime,
+}
+
+impl Default for Runtimes {
+    fn default() -> Self {
+        let tokio_runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .unwrap();
+
+        Self { tokio_runtime }
+    }
+}
+
+static RUNTIMES: LazyLock<Runtimes> = LazyLock::new(Runtimes::default);
+
+fn lsp_err(err: ResponseError) -> JsValue {
+    to_js_value(&err).unwrap()
+}
+
+fn lsp_serialize<T: serde::Serialize>(value: &T) -> JsValue {
+    match to_js_value(value) {
+        Ok(v) => v,
+        Err(err) => lsp_err(internal_error(err.to_string())),
+    }
+}
+
+// todo: poor performance, struct -> serde_json -> serde_wasm_bindgen ->
+// serialize -> deserialize??
+fn to_js_value<T: serde::Serialize>(value: &T) -> Result<JsValue, serde_wasm_bindgen::Error> {
+    value.serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true))
+}