feat: support for custom certificate configuration (#592)

* feat: add certificate option

* refactor: move SystemCompilerFeat into tinymist-world

* feat: move HttpRegistry as HttpsRegistry into tinymist-world

* feat: add reading pem file

* feat: update LspUniverseBuilder::build

* feat: fill missing argument of LspUniverseBuilder::build

* chore: update lock file for additional dependencies

* chore:  refine comment for certification

* refactor: simplify by new constructor

* refactor: sort arguments for threaded_http

* refactor: split https.rs from lib.rs in tinymist-world

Cargo.lock

@@ -4021,10 +4021,15 @@ dependencies = [
  "chrono",
  "clap",
  "comemo 0.4.0",
+ "dirs",
+ "flate2",
  "log",
+ "parking_lot",
  "reflexo-typst",
+ "reqwest",
  "serde",
  "serde_json",
+ "tar",
  "tinymist-assets 0.11.20 (registry+https://github.com/rust-lang/crates.io-index)",
  "typst-assets",
 ]

crates/tinymist-world/Cargo.toml

@@ -25,5 +25,11 @@ reflexo-typst.workspace = true
 tinymist-assets = { workspace = true }
 typst-assets = { workspace = true, features = ["fonts"] }
 
+dirs.workspace = true
+parking_lot.workspace = true
+flate2 = "1"
+tar = "0.4"
+reqwest = "^0.11"
+
 [lints]
 workspace = true

crates/tinymist-world/src/https.rs

@@ -0,0 +1,242 @@
+pub use reflexo_typst::font::FontResolverImpl;
+
+use std::path::Path;
+use std::{path::PathBuf, sync::Arc};
+
+use reflexo_typst::vfs::system::SystemAccessModel;
+use reflexo_typst::{CompilerFeat, CompilerUniverse, CompilerWorld};
+
+use std::sync::OnceLock;
+use reflexo_typst::typst::{
+    diag::{eco_format, EcoString},
+    syntax::package::PackageVersion,
+};
+use reqwest::{
+    blocking::Response,
+    Certificate
+};
+use log::error;
+use parking_lot::Mutex;
+use reflexo_typst::package::{DummyNotifier, Notifier, PackageError, PackageSpec, PackageRegistry};
+
+/// Compiler feature for LSP universe and worlds without typst.ts to implement more for tinymist.
+/// type trait of [`TypstSystemWorld`].
+#[derive(Debug, Clone, Copy)]
+pub struct SystemCompilerFeatExtend;
+
+impl CompilerFeat for SystemCompilerFeatExtend {
+    /// Uses [`FontResolverImpl`] directly.
+    type FontResolver = FontResolverImpl;
+    /// It accesses a physical file system.
+    type AccessModel = SystemAccessModel;
+    /// It performs native HTTP requests for fetching package data.
+    type Registry = HttpsRegistry;
+}
+
+/// The compiler universe in system environment.
+pub type TypstSystemUniverseExtend = CompilerUniverse<SystemCompilerFeatExtend>;
+/// The compiler world in system environment.
+pub type TypstSystemWorldExtend = CompilerWorld<SystemCompilerFeatExtend>;
+
+/// The http registry without typst.ts to implement more for tinymist.
+pub struct HttpsRegistry {
+    notifier: Arc<Mutex<dyn Notifier + Send>>,
+
+    packages: OnceLock<Vec<(PackageSpec, Option<EcoString>)>>,
+
+    cert_path: Option<PathBuf>,
+}
+
+impl Default for HttpsRegistry {
+    fn default() -> Self {
+        Self { 
+            notifier: Arc::new(Mutex::<DummyNotifier>::default()), 
+
+            // todo: reset cache
+            packages: OnceLock::new(), 
+
+            // Default to None
+            cert_path: None,
+        }
+    }
+}
+
+impl HttpsRegistry {
+    pub fn new(cert_path: Option<PathBuf>) -> Self {
+        Self { 
+            notifier: Arc::new(Mutex::<DummyNotifier>::default()), 
+            packages: OnceLock::new(), 
+            cert_path,
+        }
+    }
+
+    /// Get local path option 
+    pub fn local_path(&self) -> Option<Box<Path>> {
+        if let Some(data_dir) = dirs::data_dir() {
+            if data_dir.exists() {
+                return Some(data_dir.join("typst/packages").into());
+            }
+        }
+
+        None
+    }
+
+    /// Get data & cache dir
+    pub fn paths(&self) -> Vec<Box<Path>> {
+        let mut res = vec![];
+        if let Some(data_dir) = dirs::data_dir() {
+            let dir: Box<Path> = data_dir.join("typst/packages").into();
+            if dir.exists() {
+                res.push(dir);
+            }
+        }
+
+        if let Some(cache_dir) = dirs::cache_dir() {
+            let dir: Box<Path> = cache_dir.join("typst/packages").into();
+            if dir.exists() {
+                res.push(dir);
+            }
+        }
+
+        res
+    }
+
+
+    /// Make a package available in the on-disk cache.
+    pub fn prepare_package(&self, spec: &PackageSpec) -> Result<Arc<Path>, PackageError> {
+        let subdir = format!(
+            "typst/packages/{}/{}/{}",
+            spec.namespace, spec.name, spec.version
+        );
+
+        if let Some(data_dir) = dirs::data_dir() {
+            let dir = data_dir.join(&subdir);
+            if dir.exists() {
+                return Ok(dir.into());
+            }
+        }
+
+        if let Some(cache_dir) = dirs::cache_dir() {
+            let dir = cache_dir.join(&subdir);
+
+            // Download from network if it doesn't exist yet.
+            if spec.namespace == "preview" && !dir.exists() {
+                self.download_package(spec, &dir)?;
+            }
+
+            if dir.exists() {
+                return Ok(dir.into());
+            }
+        }
+
+        Err(PackageError::NotFound(spec.clone()))
+    }
+
+    /// Download a package over the network.
+    fn download_package(&self, spec: &PackageSpec, package_dir: &Path) -> Result<(), PackageError> {
+        let url = format!(
+            "https://packages.typst.org/preview/{}-{}.tar.gz",
+            spec.name, spec.version
+        );
+
+        self.notifier.lock().downloading(spec);
+        threaded_http(&url, self.cert_path.as_deref(), |resp| {
+            let reader = match resp.and_then(|r| r.error_for_status()) {
+                Ok(response) => response,
+                Err(err) if matches!(err.status().map(|s| s.as_u16()), Some(404)) => {
+                    return Err(PackageError::NotFound(spec.clone()))
+                }
+                Err(err) => return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))),
+            };
+
+            let decompressed = flate2::read::GzDecoder::new(reader);
+            tar::Archive::new(decompressed)
+                .unpack(package_dir)
+                .map_err(|err| {
+                    std::fs::remove_dir_all(package_dir).ok();
+                    PackageError::MalformedArchive(Some(eco_format!("{err}")))
+                })
+        })
+        .ok_or_else(|| PackageError::Other(Some(eco_format!("cannot spawn http thread"))))?
+    }
+}
+
+impl PackageRegistry for HttpsRegistry {
+    fn resolve(&self, spec: &PackageSpec) -> Result<std::sync::Arc<Path>, PackageError> {
+        self.prepare_package(spec)
+    }
+
+    fn packages(&self) -> &[(PackageSpec, Option<EcoString>)] {
+        self.packages.get_or_init(|| {
+            let url = "https://packages.typst.org/preview/index.json";
+
+            threaded_http(url, self.cert_path.as_deref(), |resp| {
+                let reader = match resp.and_then(|r| r.error_for_status()) {
+                    Ok(response) => response,
+                    Err(err) => {
+                        // todo: silent error
+                        error!("Failed to fetch package index: {} from {}", err, url);
+                        return vec![];
+                    }
+                };
+
+                #[derive(serde::Deserialize)]
+                struct RemotePackageIndex {
+                    name: EcoString,
+                    version: PackageVersion,
+                    description: Option<EcoString>,
+                }
+
+                let index: Vec<RemotePackageIndex> = match serde_json::from_reader(reader) {
+                    Ok(index) => index,
+                    Err(err) => {
+                        error!("Failed to parse package index: {} from {}", err, url);
+                        return vec![];
+                    }
+                };
+
+                index
+                    .into_iter()
+                    .map(|e| {
+                        (
+                            PackageSpec {
+                                namespace: "preview".into(),
+                                name: e.name,
+                                version: e.version,
+                            },
+                            e.description,
+                        )
+                    })
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default()
+        })
+    }
+}
+
+fn threaded_http<T: Send + Sync>(
+    url: &str,
+    cert_path: Option<&Path>,
+    f: impl FnOnce(Result<Response, reqwest::Error>) -> T + Send + Sync,
+) -> Option<T> {
+    std::thread::scope(|s| {
+                s.spawn(move || {
+            let client_builder = reqwest::blocking::Client::builder();
+
+            let client = if let Some(cert_path) = cert_path {
+                let cert = std::fs::read(cert_path).ok().and_then(|buf| Certificate::from_pem(&buf).ok());
+                if let Some(cert) = cert {
+                    client_builder.add_root_certificate(cert).build().unwrap()
+                } else {
+                    client_builder.build().unwrap()
+                }
+            } else {
+                client_builder.build().unwrap()
+            };
+
+            f(client.get(url).send())
+        })
+        .join()
+        .ok()
+    })
+}

crates/tinymist-world/src/lib.rs

@@ -16,11 +16,13 @@ use comemo::Prehashed;
 use reflexo_typst::error::prelude::*;
 use reflexo_typst::font::system::SystemFontSearcher;
 use reflexo_typst::foundations::{Str, Value};
-use reflexo_typst::package::http::HttpRegistry;
 use reflexo_typst::vfs::{system::SystemAccessModel, Vfs};
-use reflexo_typst::{SystemCompilerFeat, TypstDict, TypstSystemUniverse, TypstSystemWorld};
+use reflexo_typst::TypstDict;
 use serde::{Deserialize, Serialize};
 
+mod https;
+use https::{SystemCompilerFeatExtend, TypstSystemUniverseExtend, TypstSystemWorldExtend, HttpsRegistry};
+
 const ENV_PATH_SEP: char = if cfg!(windows) { ';' } else { ':' };
 
 /// The font arguments for the compiler.
@@ -78,6 +80,14 @@ pub struct CompileOnceArgs {
         hide(true),
     )]
     pub creation_timestamp: Option<DateTime<Utc>>,
+
+    /// Path to CA certificate file for network access, especially for downloading typst packages.
+    #[clap(
+        long = "cert",
+        env = "TYPST_CERT",
+        value_name = "CERT_PATH"
+    )]
+    pub certification: Option<PathBuf>,
 }
 
 impl CompileOnceArgs {
@@ -90,8 +100,9 @@ impl CompileOnceArgs {
             .iter()
             .map(|(k, v)| (Str::from(k.as_str()), Value::Str(Str::from(v.as_str()))))
             .collect();
+        let cert_path = self.certification.clone();
 
-        LspUniverseBuilder::build(entry, Arc::new(fonts), Arc::new(Prehashed::new(inputs)))
+        LspUniverseBuilder::build(entry, Arc::new(fonts), Arc::new(Prehashed::new(inputs)), cert_path)
             .context("failed to create universe")
     }
 
@@ -136,11 +147,11 @@ impl CompileOnceArgs {
 }
 
 /// Compiler feature for LSP universe and worlds.
-pub type LspCompilerFeat = SystemCompilerFeat;
+pub type LspCompilerFeat = SystemCompilerFeatExtend;
 /// LSP universe that spawns LSP worlds.
-pub type LspUniverse = TypstSystemUniverse;
+pub type LspUniverse = TypstSystemUniverseExtend;
 /// LSP world.
-pub type LspWorld = TypstSystemWorld;
+pub type LspWorld = TypstSystemWorldExtend;
 /// Immutable prehashed reference to dictionary.
 pub type ImmutDict = Arc<Prehashed<TypstDict>>;
 
@@ -154,12 +165,13 @@ impl LspUniverseBuilder {
         entry: EntryState,
         font_resolver: Arc<FontResolverImpl>,
         inputs: ImmutDict,
+        cert_path: Option<PathBuf>,
     ) -> ZResult<LspUniverse> {
         Ok(LspUniverse::new_raw(
             entry,
             Some(inputs),
             Vfs::new(SystemAccessModel {}),
-            HttpRegistry::default(),
+            HttpsRegistry::new(cert_path),
             font_resolver,
         ))
     }

crates/tinymist/src/actor/mod.rs

@@ -119,11 +119,12 @@ impl LanguageState {
         let entry_ = entry.clone();
         let compile_handle = handle.clone();
         let cache = self.cache.clone();
+        let cert_path = self.compile_config().determine_certification_path();
 
         self.client.handle.spawn_blocking(move || {
             // Create the world
             let font_resolver = font_resolver.wait().clone();
-            let verse = LspUniverseBuilder::build(entry_.clone(), font_resolver, inputs)
+            let verse = LspUniverseBuilder::build(entry_.clone(), font_resolver, inputs, cert_path)
                 .expect("incorrect options");
 
             // Create the actor

crates/tinymist/src/init.rs

@@ -516,6 +516,7 @@ impl CompileConfig {
                 inputs: Arc::new(Prehashed::new(inputs)),
                 font: command.font,
                 creation_timestamp: command.creation_timestamp,
+                cert: command.certification,
             });
         }
 
@@ -691,6 +692,12 @@ impl CompileConfig {
     pub fn determine_creation_timestamp(&self) -> Option<chrono::DateTime<chrono::Utc>> {
         self.typst_extra_args.as_ref()?.creation_timestamp
     }
+    
+    /// Determines the certification path.
+    pub fn determine_certification_path(&self) -> Option<PathBuf> {
+        let extras = self.typst_extra_args.as_ref()?;
+        extras.cert.clone()
+    }
 
     fn determine_user_inputs(&self) -> ImmutDict {
         static EMPTY: Lazy<ImmutDict> = Lazy::new(ImmutDict::default);
@@ -793,6 +800,8 @@ pub struct CompileExtraOpts {
     pub font: CompileFontArgs,
     /// The creation timestamp for various output.
     pub creation_timestamp: Option<chrono::DateTime<chrono::Utc>>,
+    /// Path to certification file
+    pub cert: Option<PathBuf>,
 }
 
 /// The path pattern that could be substituted.

crates/typlite/src/tests.rs

@@ -24,6 +24,7 @@ fn conv(s: &str) -> EcoString {
         EntryState::new_rooted(cwd.as_path().into(), Some(main.id())),
         font_resolver.unwrap(),
         Default::default(),
+        Default::default()
     )
     .unwrap();
     universe