VSCode web extension: load files from the vscode API

We can't use file system call to load files with wasm, we need to go through the vscode API

Note that this is all async, so i had to prevent re-entry within the compiler using a rentry trick.

I removed the lifetime in the TypeLoader because I thought this was going to be necessary
to get 'static futures. But it turned out not to be necessary. Anyway, I kept it htis way because
it is actually nicer IMHO, even if the CompilationConfiguration is now copied

editor/vscode/browser-language-server/browserServerMain.ts

@@ -21,21 +21,25 @@ slint_init(slint_wasm_data).then((_) => {
         return true;
     }
 
+    async function load_file(path: string): Promise<Uint8Array> {
+        return await connection.sendRequest("slint/load_file", path);
+    }
+
     connection.onInitialize((params: InitializeParams): InitializeResult => {
-        the_lsp = slint_lsp.create(params, send_notification);
+        the_lsp = slint_lsp.create(params, send_notification, load_file);
         return { capabilities: the_lsp.capabilities() };
     });
 
-    connection.onRequest((method, params, token) => {
-        return the_lsp.handle_request(token, method, params);
+    connection.onRequest(async (method, params, token) => {
+        return await the_lsp.handle_request(token, method, params);
     });
 
-    connection.onDidChangeTextDocument((param) => {
-        the_lsp.reload_document(param.contentChanges[param.contentChanges.length - 1].text, param.textDocument.uri);
+    connection.onDidChangeTextDocument(async (param) => {
+        await the_lsp.reload_document(param.contentChanges[param.contentChanges.length - 1].text, param.textDocument.uri);
     });
 
-    connection.onDidOpenTextDocument((param) => {
-        the_lsp.reload_document(param.textDocument.text, param.textDocument.uri);
+    connection.onDidOpenTextDocument(async (param) => {
+        await the_lsp.reload_document(param.textDocument.text, param.textDocument.uri);
     });
 
     // Listen on the connection

editor/vscode/src/browser.ts

@@ -35,6 +35,9 @@ function startClient(context: vscode.ExtensionContext) {
             context.subscriptions.push(disposable);
 
             client.onReady().then(() => {
+                client.onRequest("slint/load_file", async (param: string) => {
+                    return await vscode.workspace.fs.readFile(Uri.parse(param));
+                });
                 //client.onNotification(serverStatus, (params) => setServerStatus(params, statusBar));
             });
         }

internal/compiler/lib.rs

@@ -137,8 +137,11 @@ pub async fn compile_syntax_node(
 
     let doc_node: parser::syntax_nodes::Document = doc_node.into();
 
-    let mut loader =
-        typeloader::TypeLoader::new(global_type_registry, &compiler_config, &mut diagnostics);
+    let mut loader = typeloader::TypeLoader::new(
+        global_type_registry,
+        compiler_config.clone(),
+        &mut diagnostics,
+    );
 
     if diagnostics.has_error() {
         return (crate::object_tree::Document::default(), diagnostics);

internal/compiler/lookup.rs

@@ -38,7 +38,7 @@ pub struct LookupCtx<'a> {
 
     /// The type loader instance, which may be used to resolve relative path references
     /// for example for img!
-    pub type_loader: Option<&'a crate::typeloader::TypeLoader<'a>>,
+    pub type_loader: Option<&'a crate::typeloader::TypeLoader>,
 
     /// The token currently processed
     pub current_token: Option<NodeOrToken>,

internal/compiler/passes.rs

@@ -47,7 +47,7 @@ use crate::langtype::Type;
 pub async fn run_passes(
     doc: &crate::object_tree::Document,
     diag: &mut crate::diagnostics::BuildDiagnostics,
-    type_loader: &mut crate::typeloader::TypeLoader<'_>,
+    type_loader: &mut crate::typeloader::TypeLoader,
     compiler_config: &crate::CompilerConfiguration,
 ) {
     if matches!(doc.root_component.root_element.borrow().base_type, Type::Invalid | Type::Void) {
@@ -216,7 +216,7 @@ pub async fn run_passes(
 /// Run the passes on imported documents
 pub fn run_import_passes(
     doc: &crate::object_tree::Document,
-    type_loader: &crate::typeloader::TypeLoader<'_>,
+    type_loader: &crate::typeloader::TypeLoader,
     diag: &mut crate::diagnostics::BuildDiagnostics,
 ) {
     infer_aliases_types::resolve_aliases(doc, diag);

internal/compiler/passes/lower_layout.rs

@@ -19,7 +19,7 @@ use std::rc::Rc;
 
 pub async fn lower_layouts(
     component: &Rc<Component>,
-    type_loader: &mut TypeLoader<'_>,
+    type_loader: &mut TypeLoader,
     diag: &mut BuildDiagnostics,
 ) {
     // Ignore import errors

internal/compiler/passes/lower_tabwidget.rs

@@ -17,7 +17,7 @@ use std::rc::Rc;
 
 pub async fn lower_tabwidget(
     component: &Rc<Component>,
-    type_loader: &mut crate::typeloader::TypeLoader<'_>,
+    type_loader: &mut crate::typeloader::TypeLoader,
     diag: &mut BuildDiagnostics,
 ) {
     // Ignore import errors

internal/compiler/typeloader.rs

@@ -1,7 +1,6 @@
 // Copyright © SixtyFPS GmbH <info@slint-ui.com>
 // SPDX-License-Identifier: GPL-3.0-only OR LicenseRef-Slint-commercial
 
-use std::borrow::Cow;
 use std::cell::RefCell;
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::path::{Path, PathBuf};
@@ -58,26 +57,25 @@ impl ImportedName {
     }
 }
 
-pub struct TypeLoader<'a> {
+pub struct TypeLoader {
     pub global_type_registry: Rc<RefCell<TypeRegister>>,
-    pub compiler_config: &'a CompilerConfiguration,
-    style: Cow<'a, str>,
+    pub compiler_config: CompilerConfiguration,
+    style: String,
     all_documents: LoadedDocuments,
 }
 
-impl<'a> TypeLoader<'a> {
+impl TypeLoader {
     pub fn new(
         global_type_registry: Rc<RefCell<TypeRegister>>,
-        compiler_config: &'a CompilerConfiguration,
+        compiler_config: CompilerConfiguration,
         diag: &mut BuildDiagnostics,
     ) -> Self {
         let style = compiler_config
         .style
-        .as_ref()
-        .map(Cow::from)
-        .or_else(|| std::env::var("SLINT_STYLE").map(Cow::from).ok())
+        .clone()
+        .or_else(|| std::env::var("SLINT_STYLE").ok())
         .or_else(|| {
-            let legacy_fallback = std::env::var("SIXTYFPS_STYLE").map(Cow::from).ok();
+            let legacy_fallback = std::env::var("SIXTYFPS_STYLE").ok();
             if legacy_fallback.is_some() {
                 diag.push_diagnostic_with_span(
                     "Using `SIXTYFPS_STYLE` environment variable for dynamic backend selection. This is deprecated, use `SLINT_STYLE` instead".to_owned(),
@@ -96,7 +94,7 @@ impl<'a> TypeLoader<'a> {
                     crate::diagnostics::DiagnosticLevel::Warning
                 );
             }
-            Cow::from("fluent")
+            String::from("fluent")
         });
 
         let myself = Self {
@@ -501,7 +499,7 @@ fn test_dependency_loading() {
 
     let mut build_diagnostics = BuildDiagnostics::default();
 
-    let mut loader = TypeLoader::new(global_registry, &compiler_config, &mut build_diagnostics);
+    let mut loader = TypeLoader::new(global_registry, compiler_config, &mut build_diagnostics);
 
     spin_on::spin_on(loader.load_dependencies_recursively(
         &doc_node,
@@ -547,7 +545,7 @@ X := XX {}
     let global_registry = TypeRegister::builtin();
     let registry = Rc::new(RefCell::new(TypeRegister::new(&global_registry)));
     let mut build_diagnostics = BuildDiagnostics::default();
-    let mut loader = TypeLoader::new(global_registry, &compiler_config, &mut build_diagnostics);
+    let mut loader = TypeLoader::new(global_registry, compiler_config, &mut build_diagnostics);
     spin_on::spin_on(loader.load_dependencies_recursively(
         &doc_node,
         &mut build_diagnostics,
@@ -565,7 +563,7 @@ fn test_manual_import() {
     compiler_config.style = Some("fluent".into());
     let global_registry = TypeRegister::builtin();
     let mut build_diagnostics = BuildDiagnostics::default();
-    let mut loader = TypeLoader::new(global_registry, &compiler_config, &mut build_diagnostics);
+    let mut loader = TypeLoader::new(global_registry, compiler_config, &mut build_diagnostics);
 
     let maybe_button_type =
         spin_on::spin_on(loader.import_type("std-widgets.slint", "Button", &mut build_diagnostics));
@@ -588,7 +586,7 @@ fn test_builtin_style() {
 
     let global_registry = TypeRegister::builtin();
     let mut build_diagnostics = BuildDiagnostics::default();
-    let _loader = TypeLoader::new(global_registry, &compiler_config, &mut build_diagnostics);
+    let _loader = TypeLoader::new(global_registry, compiler_config, &mut build_diagnostics);
 
     assert!(!build_diagnostics.has_error());
 }
@@ -607,7 +605,7 @@ fn test_user_style() {
 
     let global_registry = TypeRegister::builtin();
     let mut build_diagnostics = BuildDiagnostics::default();
-    let _loader = TypeLoader::new(global_registry, &compiler_config, &mut build_diagnostics);
+    let _loader = TypeLoader::new(global_registry, compiler_config, &mut build_diagnostics);
 
     assert!(!build_diagnostics.has_error());
 }
@@ -626,7 +624,7 @@ fn test_unknown_style() {
 
     let global_registry = TypeRegister::builtin();
     let mut build_diagnostics = BuildDiagnostics::default();
-    let _loader = TypeLoader::new(global_registry, &compiler_config, &mut build_diagnostics);
+    let _loader = TypeLoader::new(global_registry, compiler_config, &mut build_diagnostics);
 
     assert!(build_diagnostics.has_error());
     let diags = build_diagnostics.to_string_vec();

tools/lsp/Cargo.toml

@@ -58,6 +58,7 @@ spin_on = "0.1"
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 wasm-bindgen = { version = "0.2.80", features = ["serde-serialize"] }
+wasm-bindgen-futures = "0.4.30"
 js-sys = { version = "0.3.57" }
 console_error_panic_hook = "0.1.5"
 

tools/lsp/main.rs

@@ -130,7 +130,7 @@ fn main_loop(connection: &Connection, params: serde_json::Value) -> Result<(), E
         Some(if cli_args.style.is_empty() { "fluent".into() } else { cli_args.style });
     compiler_config.include_paths = cli_args.include_paths;
 
-    let mut document_cache = DocumentCache::new(&compiler_config);
+    let mut document_cache = DocumentCache::new(compiler_config);
     for msg in &connection.receiver {
         match msg {
             Message::Request(req) => {
@@ -160,21 +160,21 @@ pub fn handle_notification(
     match &*req.method {
         DidOpenTextDocument::METHOD => {
             let params: DidOpenTextDocumentParams = serde_json::from_value(req.params)?;
-            reload_document(
+            spin_on::spin_on(reload_document(
                 &ServerNotifier(connection.sender.clone()),
                 params.text_document.text,
                 params.text_document.uri,
                 document_cache,
-            )?;
+            ))?;
         }
         DidChangeTextDocument::METHOD => {
             let mut params: DidChangeTextDocumentParams = serde_json::from_value(req.params)?;
-            reload_document(
+            spin_on::spin_on(reload_document(
                 &ServerNotifier(connection.sender.clone()),
                 params.content_changes.pop().unwrap().text,
                 params.text_document.uri,
                 document_cache,
-            )?;
+            ))?;
         }
         "slint/showPreview" => {
             show_preview_command(

tools/lsp/server_loop.rs

@@ -28,19 +28,19 @@ pub type Error = Box<dyn std::error::Error>;
 
 const SHOW_PREVIEW_COMMAND: &str = "showPreview";
 
-pub struct DocumentCache<'a> {
-    pub(crate) documents: TypeLoader<'a>,
-    newline_offsets: HashMap<Url, Vec<u32>>,
+pub struct DocumentCache {
+    pub(crate) documents: TypeLoader,
+    pub(crate) newline_offsets: HashMap<Url, Vec<u32>>,
 }
 
-impl<'a> DocumentCache<'a> {
-    pub fn new(config: &'a CompilerConfiguration) -> Self {
+impl DocumentCache {
+    pub fn new(config: CompilerConfiguration) -> Self {
         let documents =
             TypeLoader::new(TypeRegister::builtin(), config, &mut BuildDiagnostics::default());
         Self { documents, newline_offsets: Default::default() }
     }
 
-    fn newline_offsets_from_content(content: &str) -> Vec<u32> {
+    pub(crate) fn newline_offsets_from_content(content: &str) -> Vec<u32> {
         let mut ln_offs = 0;
         content
             .split('\n')
@@ -283,7 +283,7 @@ fn maybe_goto_preview(
     }
 }
 
-pub fn reload_document(
+pub async fn reload_document(
     connection: &crate::ServerNotifier,
     content: String,
     uri: lsp_types::Url,
@@ -297,13 +297,7 @@ pub fn reload_document(
     #[cfg(not(target_arch = "wasm32"))]
     crate::preview::set_contents(&path_canon, content.clone());
     let mut diag = BuildDiagnostics::default();
-    spin_on::spin_on(document_cache.documents.load_file(
-        &path_canon,
-        &path,
-        content,
-        false,
-        &mut diag,
-    ));
+    document_cache.documents.load_file(&path_canon, &path, content, false, &mut diag).await;
 
     // Always provide diagnostics for all files. Empty diagnostics clear any previous ones.
     let mut lsp_diags: HashMap<Url, Vec<lsp_types::Diagnostic>> = core::iter::once(&path)
@@ -329,7 +323,6 @@ pub fn reload_document(
             PublishDiagnosticsParams { uri, diagnostics, version: None },
         )?;
     }
-
     Ok(())
 }
 

tools/lsp/wasm_main.rs

@@ -11,16 +11,19 @@ mod server_loop;
 mod util;
 
 use i_slint_compiler::CompilerConfiguration;
+use js_sys::Function;
 use lsp_types::InitializeParams;
 use serde::Serialize;
 pub use server_loop::{DocumentCache, Error};
 use std::cell::RefCell;
+use std::io::ErrorKind;
 use std::rc::Rc;
 use wasm_bindgen::prelude::*;
 
 pub mod wasm_prelude {
     use std::path::{Path, PathBuf};
 
+    /// lsp_url doesn't have method to convert to and from PathBuf for wasm, so just make some
     pub trait UrlWasm {
         fn to_file_path(&self) -> Result<PathBuf, ()>;
         fn from_file_path<P: AsRef<Path>>(path: P) -> Result<lsp_types::Url, ()>;
@@ -36,7 +39,7 @@ pub mod wasm_prelude {
 }
 
 #[derive(Clone)]
-pub struct ServerNotifier(js_sys::Function);
+pub struct ServerNotifier(Function);
 impl ServerNotifier {
     pub fn send_notification(&self, method: String, params: impl Serialize) -> Result<(), Error> {
         self.0
@@ -79,32 +82,82 @@ impl RequestHolder {
     }
 }
 
+#[derive(Default)]
+struct ReentryGuard {
+    locked: bool,
+    waker: Vec<std::task::Waker>,
+}
+
+impl ReentryGuard {
+    pub async fn lock(this: Rc<RefCell<Self>>) -> ReentryGuardLock {
+        struct ReentryGuardLocker(Rc<RefCell<ReentryGuard>>);
+
+        impl std::future::Future for ReentryGuardLocker {
+            type Output = ReentryGuardLock;
+            fn poll(
+                self: std::pin::Pin<&mut Self>,
+                cx: &mut std::task::Context<'_>,
+            ) -> std::task::Poll<Self::Output> {
+                let mut s = self.0.borrow_mut();
+                if s.locked {
+                    s.waker.push(cx.waker().clone());
+                    std::task::Poll::Pending
+                } else {
+                    s.locked = true;
+                    std::task::Poll::Ready(ReentryGuardLock(self.0.clone()))
+                }
+            }
+        }
+        ReentryGuardLocker(this).await
+    }
+}
+
+struct ReentryGuardLock(Rc<RefCell<ReentryGuard>>);
+
+impl Drop for ReentryGuardLock {
+    fn drop(&mut self) {
+        let mut s = self.0.borrow_mut();
+        s.locked = false;
+        let wakers = std::mem::take(&mut s.waker);
+        drop(s);
+        for w in wakers {
+            w.wake()
+        }
+    }
+}
+
 #[wasm_bindgen]
 pub struct SlintServer {
-    document_cache: Rc<RefCell<DocumentCache<'static>>>,
+    document_cache: Rc<RefCell<DocumentCache>>,
     init_param: InitializeParams,
     notifier: ServerNotifier,
+    reentry_guard: Rc<RefCell<ReentryGuard>>,
 }
 
 #[wasm_bindgen]
 pub fn create(
     init_param: JsValue,
-    send_notification: js_sys::Function,
+    send_notification: Function,
+    load_file: Function,
 ) -> Result<SlintServer, JsError> {
     console_error_panic_hook::set_once();
 
     let init_param = init_param.into_serde()?;
 
-    let compiler_config =
+    let mut compiler_config =
         CompilerConfiguration::new(i_slint_compiler::generator::OutputFormat::Interpreter);
+    compiler_config.open_import_fallback = Some(Rc::new(move |path| {
+        let load_file = load_file.clone();
+        Box::pin(async move { Some(self::load_file(path, &load_file).await) })
+    }));
 
-    // FIXME: we leak one compiler_config
-    let document_cache = DocumentCache::new(Box::leak(Box::new(compiler_config)));
+    let document_cache = DocumentCache::new(compiler_config);
 
     Ok(SlintServer {
         document_cache: Rc::new(RefCell::new(document_cache)),
         init_param,
         notifier: ServerNotifier(send_notification),
+        reentry_guard: Default::default(),
     })
 }
 
@@ -116,15 +169,18 @@ impl SlintServer {
     }
 
     #[wasm_bindgen]
-    pub fn reload_document(&self, content: String, uri: JsValue) -> Result<(), JsError> {
-        server_loop::reload_document(
-            &self.notifier,
-            content,
-            uri.into_serde()?,
-            &mut self.document_cache.borrow_mut(),
-        )
-        .map_err(|e| JsError::new(&e.to_string()))?;
-        Ok(())
+    pub fn reload_document(&self, content: String, uri: JsValue) -> js_sys::Promise {
+        let document_cache = self.document_cache.clone();
+        let notifier = self.notifier.clone();
+        let guard = self.reentry_guard.clone();
+        wasm_bindgen_futures::future_to_promise(async move {
+            let _lock = ReentryGuard::lock(guard).await;
+            let uri: lsp_types::Url = uri.into_serde().map_err(|e| JsError::new(&e.to_string()))?;
+            server_loop::reload_document(&notifier, content, uri, &mut document_cache.borrow_mut())
+                .await
+                .map_err(|e| JsError::new(&e.to_string()))?;
+            Ok(JsValue::UNDEFINED)
+        })
     }
 
     /*  #[wasm_bindgen]
@@ -138,22 +194,40 @@ impl SlintServer {
     }*/
 
     #[wasm_bindgen]
-    pub fn handle_request(
-        &self,
-        _id: JsValue,
-        method: String,
-        params: JsValue,
-    ) -> Result<JsValue, JsError> {
-        let req = Request { method, params: params.into_serde()? };
-        let result = Rc::new(RefCell::new(None));
-        server_loop::handle_request(
-            RequestHolder { req, reply: result.clone(), notifier: self.notifier.clone() },
-            &self.init_param,
-            &mut self.document_cache.borrow_mut(),
-        )
-        .map_err(|e| JsError::new(&e.to_string()))?;
+    pub fn handle_request(&self, _id: JsValue, method: String, params: JsValue) -> js_sys::Promise {
+        let document_cache = self.document_cache.clone();
+        let notifier = self.notifier.clone();
+        let guard = self.reentry_guard.clone();
+        let init_param = self.init_param.clone();
+        wasm_bindgen_futures::future_to_promise(async move {
+            let _lock = ReentryGuard::lock(guard).await;
+            let req = Request {
+                method,
+                params: params
+                    .into_serde()
+                    .map_err(|x| format!("invalid param to handle_request: {x:?}"))?,
+            };
+            let result = Rc::new(RefCell::new(None));
+            server_loop::handle_request(
+                RequestHolder { req, reply: result.clone(), notifier: notifier.clone() },
+                &init_param,
+                &mut document_cache.borrow_mut(),
+            )
+            .map_err(|e| JsError::new(&e.to_string()))?;
 
-        let result = result.borrow_mut().take();
-        Ok(result.ok_or(JsError::new("Empty reply".into()))?)
+            let result = result.borrow_mut().take();
+            Ok(result.ok_or(JsError::new("Empty reply".into()))?)
+        })
     }
 }
+
+async fn load_file(path: String, load_file: &Function) -> std::io::Result<String> {
+    let value = load_file
+        .call1(&JsValue::UNDEFINED, &path.into())
+        .map_err(|x| std::io::Error::new(ErrorKind::Other, format!("{x:?}")))?;
+    let array = wasm_bindgen_futures::JsFuture::from(js_sys::Promise::from(value))
+        .await
+        .map_err(|e| std::io::Error::new(ErrorKind::Other, format!("{e:?}")))?;
+    String::from_utf8(js_sys::Uint8Array::from(array).to_vec())
+        .map_err(|e| std::io::Error::new(ErrorKind::InvalidData, e.to_string()))
+}