From b239224a63b8d63e82d5f65eec1e61b439841056 Mon Sep 17 00:00:00 2001 From: ParaN3xus <136563585+ParaN3xus@users.noreply.github.com> Date: Wed, 1 Oct 2025 16:40:21 +0800 Subject: [PATCH] feat: add js package registry support for tinymist-wasm (#2102) Co-authored-by: Myriad-Dreamin --- Cargo.lock | 4 + crates/sync-lsp/src/server.rs | 99 ++++++----- crates/sync-lsp/src/server/lsp_srv.rs | 13 +- .../tinymist-package/src/registry/browser.rs | 14 ++ crates/tinymist-project/Cargo.toml | 8 +- crates/tinymist-project/src/lsp.rs | 34 +++- crates/tinymist/Cargo.toml | 5 + crates/tinymist/src/project.rs | 15 ++ crates/tinymist/src/server.rs | 7 + crates/tinymist/src/web.rs | 158 +++++++++++++++--- 10 files changed, 278 insertions(+), 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8e73b3d3..1d7ca645 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4162,6 +4162,7 @@ dependencies = [ "reflexo-vec2svg", "rpds", "serde", + "serde-wasm-bindgen", "serde_json", "serde_yaml", "strum", @@ -4195,6 +4196,7 @@ dependencies = [ "vergen", "walkdir", "wasm-bindgen", + "wasm-bindgen-futures", ] [[package]] @@ -4454,6 +4456,7 @@ dependencies = [ "comemo", "dirs", "ecow", + "js-sys", "log", "notify", "parking_lot", @@ -4471,6 +4474,7 @@ dependencies = [ "toml", "typst", "typst-assets", + "wasm-bindgen", ] [[package]] diff --git a/crates/sync-lsp/src/server.rs b/crates/sync-lsp/src/server.rs index 2d4258eb..5f371801 100644 --- a/crates/sync-lsp/src/server.rs +++ b/crates/sync-lsp/src/server.rs @@ -207,14 +207,18 @@ impl LspClientRoot { /// Creates a new language server host from js. #[cfg(feature = "web")] - pub fn new_js(handle: tokio::runtime::Handle, transport: JsTransportSender) -> Self { + pub fn new_js(handle: tokio::runtime::Handle, sender: JsTransportSender) -> Self { let dummy = dummy_transport::(); let _strong = Arc::new(dummy.sender.into()); let weak = LspClient { handle, msg_kind: LspMessage::MESSAGE_KIND, - sender: TransportHost::Js(transport), + sender: TransportHost::Js { + event_id: Arc::new(AtomicU32::new(0)), + events: Arc::new(Mutex::new(HashMap::new())), + sender, + }, req_queue: Arc::new(Mutex::new(ReqQueue::default())), hook: Arc::new(()), @@ -237,45 +241,44 @@ impl LspClientRoot { type ReqHandler = Box FnOnce(&'a mut dyn Any, LspOrDapResponse) + Send + Sync>; type ReqQueue = req_queue::ReqQueue<(String, Instant), ReqHandler>; +/// Different transport mechanisms for communication. #[derive(Debug, Clone)] -enum TransportHost { +pub enum TransportHost { + /// System-level transport using native OS capabilities. System(SystemTransportSender), + /// JavaScript/WebAssembly transport for web environments. #[cfg(feature = "web")] - Js(JsTransportSender), + Js { + /// Atomic counter for generating unique event identifiers. + event_id: Arc, + /// Thread-safe storage for pending events indexed by their IDs. + events: Arc>>, + /// The actual sender implementation for JavaScript environments. + sender: JsTransportSender, + }, } +/// A sender implementation for system-level transport operations. #[derive(Debug, Clone)] -struct SystemTransportSender { +pub struct SystemTransportSender { + /// Weak reference to the connection transmitter. pub(crate) sender: Weak, } /// Creates a new js transport host. #[cfg(feature = "web")] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "camelCase")] pub struct JsTransportSender { - event_id: Arc, - events: Arc>>, - pub(crate) sender_event: js_sys::Function, - pub(crate) sender_request: js_sys::Function, - pub(crate) sender_notification: js_sys::Function, -} - -#[cfg(feature = "web")] -impl JsTransportSender { - /// Creates a new JS transport host. - pub fn new( - sender_event: js_sys::Function, - sender_request: js_sys::Function, - sender_notification: js_sys::Function, - ) -> Self { - Self { - event_id: Arc::new(AtomicU32::new(0)), - events: Arc::new(Mutex::new(HashMap::new())), - sender_event, - sender_request, - sender_notification, - } - } + #[serde(with = "serde_wasm_bindgen::preserve")] + pub(crate) send_event: js_sys::Function, + #[serde(with = "serde_wasm_bindgen::preserve")] + pub(crate) send_request: js_sys::Function, + #[serde(with = "serde_wasm_bindgen::preserve")] + pub(crate) send_notification: js_sys::Function, + /// The acutal resolving function in JavaScript + #[serde(with = "serde_wasm_bindgen::preserve")] + pub resolve_fn: js_sys::Function, } #[cfg(feature = "web")] @@ -303,17 +306,19 @@ impl TransportHost { } } #[cfg(feature = "web")] - TransportHost::Js(host) => { + TransportHost::Js { + event_id, + sender, + events, + } => { let event_id = { - let event_id = host - .event_id - .fetch_add(1, std::sync::atomic::Ordering::SeqCst); - let mut lg = host.events.lock(); + let event_id = event_id.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let mut lg = events.lock(); lg.insert(event_id, Box::new(event)); js_sys::Number::from(event_id) }; - if let Err(err) = host - .sender_event + if let Err(err) = sender + .send_event .call1(&wasm_bindgen::JsValue::UNDEFINED, &event_id.into()) { log::error!("failed to send event: {err:?}"); @@ -322,6 +327,7 @@ impl TransportHost { } } + /// Sends a message. pub fn send_message(&self, response: Message) { match self { TransportHost::System(host) => { @@ -334,12 +340,12 @@ impl TransportHost { } } #[cfg(feature = "web")] - TransportHost::Js(host) => match response { + TransportHost::Js { sender, .. } => match response { #[cfg(feature = "lsp")] Message::Lsp(lsp::Message::Request(req)) => { let msg = to_js_value(&req).expect("failed to serialize request to js value"); - if let Err(err) = host - .sender_request + if let Err(err) = sender + .send_request .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg) { log::error!("failed to send request: {err:?}"); @@ -348,8 +354,8 @@ impl TransportHost { #[cfg(feature = "lsp")] Message::Lsp(lsp::Message::Notification(req)) => { let msg = to_js_value(&req).expect("failed to serialize request to js value"); - if let Err(err) = host - .sender_notification + if let Err(err) = sender + .send_notification .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg) { log::error!("failed to send request: {err:?}"); @@ -362,8 +368,8 @@ impl TransportHost { #[cfg(feature = "dap")] Message::Dap(dap::Message::Request(req)) => { let msg = to_js_value(&req).expect("failed to serialize request to js value"); - if let Err(err) = host - .sender_request + if let Err(err) = sender + .send_request .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg) { log::error!("failed to send request: {err:?}"); @@ -372,8 +378,8 @@ impl TransportHost { #[cfg(feature = "dap")] Message::Dap(dap::Message::Event(req)) => { let msg = to_js_value(&req).expect("failed to serialize request to js value"); - if let Err(err) = host - .sender_notification + if let Err(err) = sender + .send_notification .call1(&wasm_bindgen::JsValue::UNDEFINED, &msg) { log::error!("failed to send request: {err:?}"); @@ -404,7 +410,8 @@ pub struct LspClient { pub handle: tokio::runtime::Handle, pub(crate) msg_kind: MessageKind, - sender: TransportHost, + /// The TransportHost between LspClient and LspServer + pub sender: TransportHost, pub(crate) req_queue: Arc>, pub(crate) hook: Arc, diff --git a/crates/sync-lsp/src/server/lsp_srv.rs b/crates/sync-lsp/src/server/lsp_srv.rs index 8494304f..1e849fba 100644 --- a/crates/sync-lsp/src/server/lsp_srv.rs +++ b/crates/sync-lsp/src/server/lsp_srv.rs @@ -260,7 +260,7 @@ where #[cfg(feature = "web")] pub fn on_server_event(&mut self, event_id: u32) { let evt = match &self.client.sender { - TransportHost::Js(sender) => sender.events.lock().remove(&event_id), + TransportHost::Js { events, .. } => events.lock().remove(&event_id), TransportHost::System(_) => { panic!("cannot send server event in system transport"); } @@ -403,4 +403,15 @@ where } } } + + /// Handles an incoming response. + pub fn on_lsp_response(&mut self, resp: lsp::Response) { + let client = self.client.clone(); + let Some(s) = self.state_mut() else { + log::warn!("server is not ready yet, while received response"); + return; + }; + + client.complete_lsp_request(s, resp) + } } diff --git a/crates/tinymist-package/src/registry/browser.rs b/crates/tinymist-package/src/registry/browser.rs index f427aeb3..6adbb821 100644 --- a/crates/tinymist-package/src/registry/browser.rs +++ b/crates/tinymist-package/src/registry/browser.rs @@ -3,6 +3,7 @@ use std::{io::Read, path::Path}; use js_sys::Uint8Array; +use tinymist_std::ImmutPath; use typst::diag::{EcoString, eco_format}; use wasm_bindgen::{JsValue, prelude::*}; @@ -120,6 +121,19 @@ impl PackageRegistry for JsRegistry { } } +impl JsRegistry { + /// Returns the path at which non-local packages should be stored when + /// downloaded. + pub fn package_cache_path(&self) -> Option<&ImmutPath> { + None + } + + /// Returns the path at which local packages are stored. + pub fn package_path(&self) -> Option<&ImmutPath> { + None + } +} + // todo /// Safety: `JsRegistry` is only used in the browser environment, and we cannot /// share data between workers. diff --git a/crates/tinymist-project/Cargo.toml b/crates/tinymist-project/Cargo.toml index 1fa82b11..4ac4dab8 100644 --- a/crates/tinymist-project/Cargo.toml +++ b/crates/tinymist-project/Cargo.toml @@ -34,7 +34,9 @@ tinymist-l10n.workspace = true toml = { workspace = true, optional = true } typst.workspace = true typst-assets.workspace = true -notify.workspace = true +notify = { workspace = true, optional = true } +js-sys = { workspace = true, optional = true } +wasm-bindgen = { workspace = true, optional = true } [features] @@ -43,8 +45,8 @@ no-content-hint = ["tinymist-task/no-content-hint"] lsp = ["toml"] # "system", -system = ["tinymist-std/system", "tinymist-world/system", "toml"] -web = ["tinymist-std/web", "tinymist-world/web"] +system = ["tinymist-std/system", "tinymist-world/system", "toml", "notify"] +web = ["tinymist-std/web", "tinymist-world/web", "js-sys", "wasm-bindgen"] [lints] workspace = true diff --git a/crates/tinymist-project/src/lsp.rs b/crates/tinymist-project/src/lsp.rs index 77bd7f1c..76644530 100644 --- a/crates/tinymist-project/src/lsp.rs +++ b/crates/tinymist-project/src/lsp.rs @@ -5,6 +5,8 @@ use tinymist_std::ImmutPath; use tinymist_std::error::prelude::*; use tinymist_task::ExportTarget; use tinymist_world::package::RegistryPathMapper; +#[cfg(all(not(feature = "system"), feature = "web"))] +use tinymist_world::package::registry::ProxyContext; use tinymist_world::vfs::Vfs; use tinymist_world::{ CompileSnapshot, CompilerFeat, CompilerUniverse, CompilerWorld, EntryOpts, EntryState, @@ -28,12 +30,11 @@ impl CompilerFeat for LspCompilerFeat { type FontResolver = FontResolverImpl; /// It accesses a physical file system. type AccessModel = DynAccessModel; - /// It performs native HTTP requests for fetching package data. - #[cfg(feature = "system")] - type Registry = tinymist_world::package::registry::HttpRegistry; - // todo: registry in browser - #[cfg(not(feature = "system"))] - type Registry = tinymist_world::package::registry::DummyRegistry; + /// It performs: + /// - native HTTP requests for fetching package data in system environment + /// - js proxied requests to browser environment + /// - no package registry in other environments + type Registry = LspRegistry; } /// LSP universe that spawns LSP worlds. @@ -211,10 +212,12 @@ impl WorldProvider for (crate::ProjectInput, ImmutPath) { } } -#[cfg(not(feature = "system"))] -type LspRegistry = tinymist_world::package::registry::DummyRegistry; +#[cfg(all(not(feature = "system"), feature = "web"))] +type LspRegistry = tinymist_world::package::registry::JsRegistry; #[cfg(feature = "system")] type LspRegistry = tinymist_world::package::registry::HttpRegistry; +#[cfg(not(any(feature = "system", feature = "web")))] +type LspRegistry = tinymist_world::package::registry::DummyRegistry; /// Builder for LSP universe. pub struct LspUniverseBuilder; @@ -318,7 +321,20 @@ impl LspUniverseBuilder { } /// Resolves package registry from given options. - #[cfg(not(feature = "system"))] + #[cfg(all(not(feature = "system"), feature = "web"))] + pub fn resolve_package( + _cert_path: Option, + _args: Option<&CompilePackageArgs>, + resolve_fn: js_sys::Function, + ) -> tinymist_world::package::registry::JsRegistry { + tinymist_world::package::registry::JsRegistry { + context: ProxyContext::new(wasm_bindgen::JsValue::NULL), + real_resolve_fn: resolve_fn, + } + } + + /// Resolves package registry from given options. + #[cfg(not(any(feature = "system", feature = "web")))] pub fn resolve_package( _cert_path: Option, _args: Option<&CompilePackageArgs>, diff --git a/crates/tinymist/Cargo.toml b/crates/tinymist/Cargo.toml index 2adffc7b..8356d838 100644 --- a/crates/tinymist/Cargo.toml +++ b/crates/tinymist/Cargo.toml @@ -90,6 +90,8 @@ hyper-tungstenite = { workspace = true, optional = true } console_error_panic_hook = { version = "0.1.2", optional = true } js-sys = { version = "0.3.77", optional = true } wasm-bindgen = { version = "0.2.100", optional = true } +serde-wasm-bindgen = { version = "0.6.5", optional = true } +wasm-bindgen-futures = { version = "0.4.50", optional = true } [dev-dependencies] temp-env.workspace = true @@ -119,8 +121,11 @@ web = [ "console_error_panic_hook", "js-sys", "wasm-bindgen", + "serde-wasm-bindgen", + "wasm-bindgen-futures", "reflexo-typst/web", "tinymist-project/web", + "sync-ls/web", ] open = ["dep:open"] system = [ diff --git a/crates/tinymist/src/project.rs b/crates/tinymist/src/project.rs index 38ed093d..4a1372a0 100644 --- a/crates/tinymist/src/project.rs +++ b/crates/tinymist/src/project.rs @@ -96,6 +96,14 @@ impl ServerState { self.dep_tx.clone(), #[cfg(feature = "preview")] self.preview.watchers.clone(), + #[cfg(all(not(feature = "system"), feature = "web"))] + if let sync_ls::TransportHost::Js { sender, .. } = + self.client.clone().to_untyped().sender + { + sender.resolve_fn + } else { + panic!("Expected Js TransportHost") + }, ); let mut old_project = std::mem::replace(&mut self.project, new_project); @@ -139,6 +147,7 @@ impl ServerState { client: TypedLspClient, dep_tx: mpsc::UnboundedSender, #[cfg(feature = "preview")] preview: ProjectPreviewState, + #[cfg(all(not(feature = "system"), feature = "web"))] resolve_fn: js_sys::Function, ) -> ProjectState { let const_config = &config.const_config; @@ -199,7 +208,13 @@ impl ServerState { log::info!("ServerState: creating ProjectState, entry: {entry:?}, inputs: {inputs:?}"); let fonts = config.fonts(); + + #[cfg(all(not(feature = "system"), feature = "web"))] + let packages = + LspUniverseBuilder::resolve_package(cert_path.clone(), Some(&package), resolve_fn); + #[cfg(any(feature = "system", not(feature = "web")))] let packages = LspUniverseBuilder::resolve_package(cert_path.clone(), Some(&package)); + let creation_timestamp = config.creation_timestamp(); let verse = LspUniverseBuilder::build( entry, diff --git a/crates/tinymist/src/server.rs b/crates/tinymist/src/server.rs index 11e8706b..8b431f6a 100644 --- a/crates/tinymist/src/server.rs +++ b/crates/tinymist/src/server.rs @@ -131,6 +131,7 @@ impl ServerState { #[cfg(feature = "preview")] let watchers = crate::project::ProjectPreviewState::default(); + let handle = Self::project( &config, editor_tx.clone(), @@ -138,6 +139,12 @@ impl ServerState { dep_tx.clone(), #[cfg(feature = "preview")] watchers.clone(), + #[cfg(all(not(feature = "system"), feature = "web"))] + if let TransportHost::Js { sender, .. } = client.clone().to_untyped().sender { + sender.resolve_fn + } else { + panic!("Expected Js TransportHost") + }, ); Self { diff --git a/crates/tinymist/src/web.rs b/crates/tinymist/src/web.rs index d7a2505d..a14c1e50 100644 --- a/crates/tinymist/src/web.rs +++ b/crates/tinymist/src/web.rs @@ -2,10 +2,18 @@ #![allow(unused)] +use std::sync::LazyLock; + +use futures::future::MaybeDone; use js_sys::{Function, Promise}; +use sync_ls::{ + internal_error, invalid_params, JsTransportSender, LsDriver, LspBuilder, LspClientRoot, + LspMessage, ResponseError, +}; +use tinymist_project::CompileFontArgs; use wasm_bindgen::prelude::*; -use crate::LONG_VERSION; +use crate::{RegularInit, ServerState, LONG_VERSION}; /// Gets the long version description of the library. #[wasm_bindgen] @@ -13,39 +21,149 @@ pub fn version() -> String { LONG_VERSION.clone() } -/// The Tinymist Language Server for WebAssembly. +/// TinymistLanguageServer implements the LSP protocol for Typst documents +/// in a WebAssembly environment #[wasm_bindgen] pub struct TinymistLanguageServer { - send_diagnostics: Function, - send_request: Function, - send_notification: Function, + /// The client root that strongly references the LSP client. + _client: LspClientRoot, + /// The mutable state of the server. + state: LsDriver, } #[wasm_bindgen] impl TinymistLanguageServer { - /// Creates a new instance of the Tinymist Language Server. + /// Creates a new language server. #[wasm_bindgen(constructor)] - pub fn new( - send_diagnostics: Function, - send_request: Function, - send_notification: Function, - ) -> Self { + pub fn new(init_opts: JsValue) -> Result { + let sender = serde_wasm_bindgen::from_value::(init_opts) + .map_err(|err| JsValue::from_str(&format!("Failed to deserialize init opts: {err}")))?; + std::panic::set_hook(Box::new(console_error_panic_hook::hook)); - Self { - send_diagnostics, - send_request, - send_notification, - } + let _client = LspClientRoot::new_js(RUNTIMES.tokio_runtime.handle().clone(), sender); + // Starts logging + let _ = crate::init_log(crate::InitLogOpts { + is_transient_cmd: false, + is_test_no_verbose: false, + output: Some(_client.weak()), + }); + let state = ServerState::install_lsp(LspBuilder::new( + RegularInit { + client: _client.weak().to_typed(), + font_opts: CompileFontArgs::default(), + exec_cmds: Vec::new(), + }, + _client.weak(), + )) + .build(); + + Ok(Self { _client, state }) + } + + /// Handles internal events. + pub fn on_event(&mut self, event_id: u32) { + self.state.on_server_event(event_id); } /// Handles incoming requests. - pub fn on_request(&self, method: String, js_params: JsValue) -> Result { - todo!() + pub fn on_request(&mut self, method: String, js_params: JsValue) -> JsValue { + let params = serde_wasm_bindgen::from_value::(js_params); + let params = match params { + Ok(p) => p, + Err(err) => return lsp_err(invalid_params(err)), + }; + + let result = self.state.on_lsp_request(&method, params); + + match result { + Ok(MaybeDone::Done(Ok(t))) => lsp_serialize(&t), + Ok(MaybeDone::Done(Err(err))) => lsp_err(err), + // tokio doesn't get scheduled anymore after returning to js world + Ok(MaybeDone::Future(fut)) => wasm_bindgen_futures::future_to_promise(async move { + Ok(match fut.await { + Ok(t) => lsp_serialize(&t), + Err(err) => lsp_err(err), + }) + }) + .into(), + // match futures::executor::block_on(fut) { + // Ok(t) => lsp_serialize(&t), + // Err(err) => lsp_err(err), + // }, + Ok(MaybeDone::Gone) => lsp_err(internal_error("response was weirdly gone")), + Err(err) => lsp_err(err), + } } /// Handles incoming notifications. - pub fn on_notification(&self, method: String, js_params: JsValue) -> Promise { - todo!() + pub fn on_notification(&mut self, method: String, js_params: JsValue) { + let params = serde_wasm_bindgen::from_value::(js_params); + let params = match params { + Ok(p) => p, + Err(err) => { + log::error!("Failed to deserialize notification params: {err}"); + return; + } + }; + + let err = self.state.on_notification(&method, params); + if let Err(err) = err { + log::error!("Failed to handle notification {method}: {err:?}"); + } + } + + /// Handles incoming responses. + pub fn on_response(&mut self, js_result: JsValue) { + let result = serde_wasm_bindgen::from_value::(js_result); + let resp = match result { + Ok(r) => r, + Err(err) => { + log::error!("Failed to deserialize response: {err}"); + return; + } + }; + self.state.on_lsp_response(resp); + } + + /// Get the version of the language server. + pub fn version() -> String { + env!("CARGO_PKG_VERSION").to_string() } } + +/// The runtimes used by the application. +pub struct Runtimes { + /// The tokio runtime. + pub tokio_runtime: tokio::runtime::Runtime, +} + +impl Default for Runtimes { + fn default() -> Self { + let tokio_runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + Self { tokio_runtime } + } +} + +static RUNTIMES: LazyLock = LazyLock::new(Runtimes::default); + +fn lsp_err(err: ResponseError) -> JsValue { + to_js_value(&err).unwrap() +} + +fn lsp_serialize(value: &T) -> JsValue { + match to_js_value(value) { + Ok(v) => v, + Err(err) => lsp_err(internal_error(err.to_string())), + } +} + +// todo: poor performance, struct -> serde_json -> serde_wasm_bindgen -> +// serialize -> deserialize?? +fn to_js_value(value: &T) -> Result { + value.serialize(&serde_wasm_bindgen::Serializer::new().serialize_maps_as_objects(true)) +}