feat: extract package implementation (#1647)

* feat: extract registry implementation

* feat: tinymist package

* fix: guard

* fix: guard 2

* feat: no specifier

* fix: temp_dir_in impl

* fix: impls

* feat: UniversePack::new

* feat: clone into memory

* feat: implement some pack for testing

* build: update cargo.lock

* feat: fit for web

* fix: guard

crates/tinymist-package/Cargo.toml

@@ -0,0 +1,65 @@
+[package]
+name = "tinymist-package"
+description = "Tinymist package support for Typst."
+categories = ["compilers"]
+keywords = ["api", "language", "typst"]
+authors.workspace = true
+version.workspace = true
+license.workspace = true
+edition.workspace = true
+homepage.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+
+base64.workspace = true
+comemo.workspace = true
+ecow.workspace = true
+flate2.workspace = true
+log.workspace = true
+parking_lot.workspace = true
+rayon.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tar.workspace = true
+typst.workspace = true
+tinymist-std.workspace = true
+
+toml = { workspace = true, optional = true }
+reqwest = { workspace = true, optional = true }
+dirs = { workspace = true, optional = true }
+walkdir = { workspace = true, optional = true }
+fastrand = { workspace = true, optional = true }
+
+js-sys = { workspace = true, optional = true }
+wasm-bindgen = { workspace = true, optional = true }
+web-sys = { workspace = true, optional = true, features = ["console"] }
+
+[dev-dependencies]
+insta.workspace = true
+
+[features]
+
+default = []
+fs-pack = ["walkdir"]
+gitcl-pack = ["toml", "tinymist-std/system"]
+http-pack = ["http-registry"]
+release-pack = ["http-pack"]
+universe-pack = ["http-pack"]
+http-registry = ["reqwest", "dirs"]
+web = ["wasm-bindgen", "web-sys", "js-sys", "tinymist-std/web"]
+browser = ["web"]
+system = [
+    "http-registry",
+    "fs-pack",
+    "gitcl-pack",
+    "http-pack",
+    "release-pack",
+    "universe-pack",
+    "tinymist-std/system",
+]
+
+
+[lints]
+workspace = true

crates/tinymist-package/src/lib.rs

@@ -0,0 +1,7 @@
+//! Package Implementation for Typst.
+
+pub mod pack;
+pub use pack::*;
+
+pub mod registry;
+pub use registry::{PackageError, PackageRegistry, PackageSpec};

crates/tinymist-package/src/pack.rs

@@ -0,0 +1,133 @@
+//! A bundle that is modifiable
+
+use core::fmt;
+use std::fmt::Display;
+use std::io::{self, Read};
+use std::path::Path;
+use std::sync::Arc;
+
+use ecow::{eco_format, EcoVec};
+use tinymist_std::{ImmutBytes, ImmutPath};
+use typst::diag::{PackageError, PackageResult};
+use typst::syntax::package::{PackageSpec, VersionlessPackageSpec};
+
+#[cfg(feature = "fs-pack")]
+mod fs;
+#[cfg(feature = "gitcl-pack")]
+mod gitcl;
+#[cfg(feature = "http-pack")]
+mod http;
+mod memory;
+mod ops;
+#[cfg(feature = "release-pack")]
+mod release;
+mod tarball;
+#[cfg(feature = "universe-pack")]
+mod universe;
+
+#[cfg(feature = "fs-pack")]
+pub use fs::*;
+#[cfg(feature = "gitcl-pack")]
+pub use gitcl::*;
+#[cfg(feature = "http-pack")]
+pub use http::*;
+pub use memory::*;
+pub use ops::*;
+#[cfg(feature = "release-pack")]
+pub use release::*;
+pub use tarball::*;
+#[cfg(feature = "universe-pack")]
+pub use universe::*;
+
+/// The pack file is the knownn file type in the package.
+pub enum PackFile<'a> {
+    /// A single file in the memory.
+    Data(io::Cursor<ImmutBytes>),
+    /// A file in the package.
+    Read(Box<dyn Read + 'a>),
+}
+
+impl io::Read for PackFile<'_> {
+    fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
+        match self {
+            PackFile::Data(data) => data.read(buf),
+            PackFile::Read(reader) => reader.read(buf),
+        }
+    }
+}
+
+/// The pack file is the knownn file type in the package.
+pub enum PackEntries<'a> {
+    /// A single file in the memory.
+    Data(EcoVec<ImmutPath>),
+    /// A file in the package.
+    Read(Box<dyn Iterator<Item = Path> + 'a>),
+}
+
+/// The pack trait is used for read/write files in a package.
+pub trait PackFs: fmt::Debug {
+    /// Read files from the package.
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()>;
+    /// Read a file from the package.
+    fn read(&self, _path: &str) -> io::Result<PackFile> {
+        Err(unsupported())
+    }
+    /// Read entries from the package.
+    fn entries(&self) -> io::Result<PackEntries> {
+        Err(unsupported())
+    }
+}
+
+/// The specifier is used to identify a package.
+pub enum PackSpecifier {
+    /// A package with a version.
+    Versioned(PackageSpec),
+    /// A package without a version.
+    Versionless(VersionlessPackageSpec),
+}
+
+/// The pack trait is used to hold a package.
+pub trait Pack: PackFs {}
+
+/// The pack trait extension.
+pub trait PackExt: Pack {
+    /// Filter the package files to read by a function.
+    fn filter(&mut self, f: impl Fn(&str) -> bool + Send + Sync) -> impl Pack
+    where
+        Self: std::marker::Sized,
+    {
+        FilterPack { src: self, f }
+    }
+}
+
+/// The pack trait is used to hold a package.
+pub trait CloneIntoPack: fmt::Debug {
+    /// Clones the pack into a new pack.
+    fn clone_into_pack(&mut self, pack: &mut impl PackFs) -> std::io::Result<()>;
+}
+
+/// The package is a trait that can be used to create a package.
+#[derive(Debug, Clone)]
+pub struct Package {
+    /// The underlying pack.
+    pub pack: Arc<dyn Pack + Send + Sync>,
+}
+
+fn unsupported() -> io::Error {
+    io::Error::new(io::ErrorKind::Unsupported, "unsupported operation")
+}
+
+fn malform(e: io::Error) -> PackageError {
+    PackageError::MalformedArchive(Some(eco_format!("{e:?}")))
+}
+
+fn other_io(e: impl Display) -> io::Error {
+    io::Error::new(io::ErrorKind::Other, e.to_string())
+}
+
+fn other(e: impl Display) -> PackageError {
+    PackageError::Other(Some(eco_format!("{e}")))
+}

crates/tinymist-package/src/pack/fs.rs

@@ -0,0 +1,111 @@
+use std::{fs::File, io::Write};
+
+use super::*;
+
+/// A package in the directory.
+#[derive(Clone)]
+pub struct DirPack<P> {
+    /// The patch storing the package.
+    pub path: P,
+}
+
+impl<P: AsRef<Path>> DirPack<P> {
+    /// Creates a new `DirPack` instance.
+    pub fn new(path: P) -> Self {
+        Self { path }
+    }
+}
+
+impl<P: AsRef<Path>> fmt::Debug for DirPack<P> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "DirPack({})", self.path.as_ref().display())
+    }
+}
+
+impl<P: AsRef<Path>> PackFs for DirPack<P> {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        self.filter(|_| true).read_all(f)
+    }
+}
+
+impl<P: AsRef<Path>> Pack for DirPack<P> {}
+impl<P: AsRef<Path>> PackExt for DirPack<P> {
+    fn filter(&mut self, f: impl Fn(&str) -> bool + Send + Sync) -> impl Pack
+    where
+        Self: std::marker::Sized,
+    {
+        FilterDirPack {
+            path: &self.path,
+            f,
+        }
+    }
+}
+
+impl<P: AsRef<Path>> CloneIntoPack for DirPack<P> {
+    fn clone_into_pack(&mut self, pack: &mut impl PackFs) -> std::io::Result<()> {
+        let base = self.path.as_ref();
+        pack.read_all(&mut |path, file| {
+            let path = base.join(path);
+            std::fs::create_dir_all(path.parent().unwrap()).map_err(other)?;
+            let mut dst = std::fs::File::create(path).map_err(other)?;
+            match file {
+                PackFile::Read(mut reader) => {
+                    std::io::copy(&mut reader, &mut dst).map_err(other)?;
+                }
+                PackFile::Data(data) => {
+                    dst.write_all(&data.into_inner()).map_err(other)?;
+                }
+            }
+
+            Ok(())
+        })
+        .map_err(other_io)?;
+        Ok(())
+    }
+}
+
+struct FilterDirPack<'a, P, F> {
+    /// The patch storing the package.
+    pub path: &'a P,
+    /// The filter function.
+    pub f: F,
+}
+
+impl<S: AsRef<Path>, F> fmt::Debug for FilterDirPack<'_, S, F> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "FilterDirPack({:?}, ..)", self.path.as_ref())
+    }
+}
+impl<Src: AsRef<Path>, F: Fn(&str) -> bool + Send + Sync> PackFs for FilterDirPack<'_, Src, F> {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        let w = walkdir::WalkDir::new(self.path.as_ref())
+            .follow_links(true)
+            .into_iter()
+            .filter_entry(|e| !e.file_name().to_string_lossy().starts_with('.'))
+            .filter_map(|e| e.ok())
+            .filter(|e| e.file_type().is_file());
+
+        for entry in w {
+            let path = entry.path();
+            let rel_path = path.strip_prefix(self.path.as_ref()).map_err(other)?;
+
+            let file_path = rel_path.to_string_lossy();
+            if !(self.f)(&file_path) {
+                continue;
+            }
+
+            let pack_file = PackFile::Read(Box::new(File::open(path).map_err(other)?));
+            f(&file_path, pack_file)?;
+        }
+
+        Ok(())
+    }
+}
+
+impl<Src: AsRef<Path>, F: Fn(&str) -> bool + Send + Sync> Pack for FilterDirPack<'_, Src, F> {}

crates/tinymist-package/src/pack/gitcl.rs

@@ -0,0 +1,63 @@
+use ecow::EcoString;
+
+use super::*;
+
+/// A package in the git.
+#[derive(Clone)]
+pub struct GitClPack<P> {
+    /// The namespace to mount.
+    pub namespace: EcoString,
+    /// The URL of the git.
+    pub url: P,
+}
+
+impl<P: AsRef<str>> GitClPack<P> {
+    /// Creates a new `GitClPack` instance.
+    pub fn new(namespace: EcoString, url: P) -> Self {
+        Self { namespace, url }
+    }
+}
+
+impl<P: AsRef<str>> fmt::Debug for GitClPack<P> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "GitClPack({})", self.url.as_ref())
+    }
+}
+
+impl<P: AsRef<str>> PackFs for GitClPack<P> {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        let temp_dir = std::env::temp_dir();
+        let temp_dir = temp_dir.join("tinymist/package-gitcl");
+
+        tinymist_std::fs::paths::temp_dir_in(temp_dir, |temp_dir| {
+            let package_path = temp_dir.join("package");
+            clone(self.url.as_ref(), &package_path)?;
+
+            Ok(DirPack::new(package_path).read_all(f))
+        })
+        .map_err(other)?
+    }
+}
+
+impl<P: AsRef<str>> Pack for GitClPack<P> {}
+impl<P: AsRef<str>> PackExt for GitClPack<P> {}
+
+fn clone(url: &str, dst: &Path) -> io::Result<()> {
+    let mut cmd = gitcl();
+    cmd.arg("clone").arg(url).arg(dst);
+    let status = cmd.status()?;
+    if !status.success() {
+        return Err(io::Error::new(
+            io::ErrorKind::Other,
+            format!("git clone failed: {status}"),
+        ));
+    }
+    Ok(())
+}
+
+fn gitcl() -> std::process::Command {
+    std::process::Command::new("git")
+}

crates/tinymist-package/src/pack/http.rs

@@ -0,0 +1,61 @@
+use ecow::eco_format;
+use typst::diag::PackageError;
+
+use super::*;
+use crate::registry::threaded_http;
+
+/// A package in the remote http.
+#[derive(Clone)]
+pub struct HttpPack<S> {
+    /// The package specifier.
+    pub specifier: PackageSpec,
+    /// The url of the package.
+    pub url: S,
+}
+
+impl<S: AsRef<str>> HttpPack<S> {
+    /// Creates a new `HttpPack` instance.
+    pub fn new(specifier: PackageSpec, url: S) -> Self {
+        Self { specifier, url }
+    }
+}
+
+impl<S: AsRef<str>> fmt::Debug for HttpPack<S> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "HttpPack({})", self.url.as_ref())
+    }
+}
+
+impl<S: AsRef<str>> PackFs for HttpPack<S> {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        let spec = &self.specifier;
+        let url = self.url.as_ref();
+        threaded_http(url, None, |resp| {
+            let reader = match resp.and_then(|r| r.error_for_status()) {
+                Ok(response) => response,
+                Err(err) if matches!(err.status().map(|s| s.as_u16()), Some(404)) => {
+                    return Err(PackageError::NotFound(spec.clone()))
+                }
+                Err(err) => return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))),
+            };
+
+            let decompressed = flate2::read::GzDecoder::new(reader);
+            let mut tarbar = TarballPack::new(decompressed);
+
+            // .unpack(package_dir)
+            // .map_err(|err| {
+            //     std::fs::remove_dir_all(package_dir).ok();
+            //     PackageError::MalformedArchive(Some(eco_format!("{err}")))
+            // })
+
+            tarbar.read_all(f)
+        })
+        .ok_or_else(|| PackageError::Other(Some(eco_format!("cannot spawn http thread"))))?
+    }
+}
+
+impl<S: AsRef<str>> Pack for HttpPack<S> {}
+impl<P: AsRef<str>> PackExt for HttpPack<P> {}

crates/tinymist-package/src/pack/memory.rs

@@ -0,0 +1,55 @@
+use std::{collections::HashMap, io::Cursor};
+
+use ecow::EcoString;
+
+use super::*;
+
+/// A package in the directory.
+#[derive(Default, Debug, Clone)]
+pub struct MapPack {
+    /// The files storing the package.
+    pub files: HashMap<EcoString, ImmutBytes>,
+}
+
+impl MapPack {
+    /// Creates a new `MapPack` instance.
+    pub fn new(files: HashMap<EcoString, ImmutBytes>) -> Self {
+        Self { files }
+    }
+}
+
+impl PackFs for MapPack {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        for (path, data) in self.files.iter() {
+            let pack_file = PackFile::Data(Cursor::new(data.clone()));
+            f(path, pack_file)?;
+        }
+
+        Ok(())
+    }
+}
+
+impl Pack for MapPack {}
+impl PackExt for MapPack {}
+
+impl CloneIntoPack for MapPack {
+    fn clone_into_pack(&mut self, pack: &mut impl PackFs) -> std::io::Result<()> {
+        pack.read_all(&mut |path, file| {
+            let data = match file {
+                PackFile::Read(mut reader) => {
+                    let mut dst = Vec::new();
+                    std::io::copy(&mut reader, &mut dst).map_err(other)?;
+                    ImmutBytes::from(dst)
+                }
+                PackFile::Data(data) => data.into_inner(),
+            };
+            self.files.insert(path.into(), data);
+            Ok(())
+        })
+        .map_err(other_io)?;
+        Ok(())
+    }
+}

crates/tinymist-package/src/pack/ops.rs

@@ -0,0 +1,31 @@
+use super::*;
+
+/// A package in the directory.
+pub struct FilterPack<'a, Src, F> {
+    /// The files storing the package.
+    pub(crate) src: &'a mut Src,
+    /// The filter function to apply to each file.
+    pub(crate) f: F,
+}
+
+impl<S: PackFs, F> fmt::Debug for FilterPack<'_, S, F> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "FilterPack({:?}, ..)", self.src)
+    }
+}
+impl<Src: PackFs, F: Fn(&str) -> bool + Send + Sync> PackFs for FilterPack<'_, Src, F> {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        self.src.read_all(&mut |path, file| {
+            if (self.f)(path) {
+                f(path, file)
+            } else {
+                Ok(())
+            }
+        })
+    }
+}
+
+impl<Src: PackFs, F: Fn(&str) -> bool + Send + Sync> Pack for FilterPack<'_, Src, F> {}

crates/tinymist-package/src/pack/release.rs

@@ -0,0 +1,31 @@
+use ecow::EcoString;
+
+use super::*;
+
+/// A package in the GitHub releases.
+#[derive(Debug, Clone)]
+pub struct GitHubReleasePack {
+    /// The package specifier.
+    pub specifier: PackageSpec,
+    /// The URL of the package.
+    pub repo: EcoString,
+    /// The name of the package.
+    pub name: EcoString,
+}
+
+impl PackFs for GitHubReleasePack {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        let url = format!(
+            "https://api.github.com/repos/{}/releases/latest/{}",
+            self.repo, self.name,
+        );
+
+        HttpPack::new(self.specifier.clone(), url).read_all(f)
+    }
+}
+
+impl Pack for GitHubReleasePack {}
+impl PackExt for GitHubReleasePack {}

crates/tinymist-package/src/pack/tarball.rs

@@ -0,0 +1,49 @@
+use super::*;
+
+/// A package in the tarball.
+pub struct TarballPack<R: ?Sized + Read> {
+    /// The holder for the tarball.
+    pub reader: tar::Archive<R>,
+}
+
+impl<R: Read> TarballPack<R> {
+    /// Creates a new `TarballPack` instance.
+    pub fn new(reader: R) -> Self {
+        let reader = tar::Archive::new(reader);
+        Self { reader }
+    }
+}
+
+impl<R: ?Sized + Read> fmt::Debug for TarballPack<R> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("TarballPack").finish()
+    }
+}
+
+impl<R: Read> PackFs for TarballPack<R> {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        for entry in self.reader.entries().map_err(malform)? {
+            let entry = entry.map_err(malform)?;
+            let header = entry.header();
+
+            let is_file = header.entry_type().is_file();
+            if !is_file {
+                continue;
+            }
+
+            let path = header.path().map_err(malform)?;
+            let path = path.to_string_lossy().to_string();
+
+            let pack_file = PackFile::Read(Box::new(entry));
+            f(&path, pack_file)?;
+        }
+
+        Ok(())
+    }
+}
+
+impl<R: Read> Pack for TarballPack<R> {}
+impl<R: Read> PackExt for TarballPack<R> {}

crates/tinymist-package/src/pack/universe.rs

@@ -0,0 +1,36 @@
+use super::*;
+use crate::registry::DEFAULT_REGISTRY;
+
+/// A package in the universe registry.
+#[derive(Debug, Clone)]
+pub struct UniversePack {
+    /// The package specifier.
+    pub specifier: PackageSpec,
+}
+
+impl UniversePack {
+    /// Creates a new `UniversePack` instance.
+    pub fn new(specifier: PackageSpec) -> Self {
+        Self { specifier }
+    }
+}
+
+impl PackFs for UniversePack {
+    fn read_all(
+        &mut self,
+        f: &mut (dyn FnMut(&str, PackFile) -> PackageResult<()> + Send + Sync),
+    ) -> PackageResult<()> {
+        let spec = &self.specifier;
+        assert_eq!(spec.namespace, "preview");
+
+        let url = format!(
+            "{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz",
+            spec.name, spec.version
+        );
+
+        HttpPack::new(self.specifier.clone(), url).read_all(f)
+    }
+}
+
+impl Pack for UniversePack {}
+impl PackExt for UniversePack {}

crates/tinymist-package/src/registry.rs

@@ -0,0 +1,62 @@
+//! Package Registry.
+
+use std::num::NonZeroUsize;
+use std::{path::Path, sync::Arc};
+
+use ecow::EcoString;
+pub use typst::diag::PackageError;
+pub use typst::syntax::package::PackageSpec;
+
+mod dummy;
+pub use dummy::*;
+
+#[cfg(feature = "browser")]
+mod browser;
+#[cfg(feature = "browser")]
+pub use browser::*;
+
+#[cfg(feature = "http-registry")]
+mod http;
+#[cfg(feature = "http-registry")]
+pub use http::*;
+
+/// The default Typst registry.
+pub const DEFAULT_REGISTRY: &str = "https://packages.typst.org";
+
+/// A trait for package registries.
+pub trait PackageRegistry {
+    /// A function to be called when the registry is reset.
+    fn reset(&mut self) {}
+
+    /// If the state of package registry can be well-defined by a revision, it
+    /// should return it. This is used to determine if the compiler should clean
+    /// and pull the registry again.
+    fn revision(&self) -> Option<NonZeroUsize> {
+        None
+    }
+
+    /// Resolves a package specification to a local path.
+    fn resolve(&self, spec: &PackageSpec) -> Result<Arc<Path>, PackageError>;
+
+    /// A list of all available packages and optionally descriptions for them.
+    ///
+    /// This function is optional to implement. It enhances the user experience
+    /// by enabling autocompletion for packages. Details about packages from the
+    /// `@preview` namespace are available from
+    /// `https://packages.typst.org/preview/index.json`.
+    fn packages(&self) -> &[(PackageSpec, Option<EcoString>)] {
+        &[]
+    }
+}
+
+/// A trait for package registries that can be notified.
+pub trait Notifier {
+    /// Called when a package is being downloaded.
+    fn downloading(&self, _spec: &PackageSpec) {}
+}
+
+/// A dummy notifier that does nothing.
+#[derive(Debug, Default, Clone, Copy, Hash)]
+pub struct DummyNotifier;
+
+impl Notifier for DummyNotifier {}

crates/tinymist-package/src/registry/browser.rs

@@ -0,0 +1,129 @@
+//! Browser proxy registry for tinymist. You should implement interfaces in js.
+
+use std::{io::Read, path::Path};
+
+use js_sys::Uint8Array;
+use typst::diag::{eco_format, EcoString};
+use wasm_bindgen::{prelude::*, JsValue};
+
+use super::{PackageError, PackageRegistry, PackageSpec};
+
+/// The `ProxyContext` struct is a wrapper around a JavaScript this.
+#[wasm_bindgen]
+#[derive(Debug, Clone)]
+pub struct ProxyContext {
+    context: JsValue,
+}
+
+#[wasm_bindgen]
+impl ProxyContext {
+    /// Creates a new `ProxyContext` instance.
+    #[wasm_bindgen(constructor)]
+    pub fn new(context: JsValue) -> Self {
+        Self { context }
+    }
+
+    /// Returns the JavaScript this.
+    #[wasm_bindgen(getter)]
+    pub fn context(&self) -> JsValue {
+        self.context.clone()
+    }
+
+    /// A convenience function to untar a tarball and call a callback for each
+    /// entry.
+    pub fn untar(&self, data: &[u8], cb: js_sys::Function) -> Result<(), JsValue> {
+        let cb = move |key: String, value: &[u8], mtime: u64| -> Result<(), JsValue> {
+            let key = JsValue::from_str(&key);
+            let value = Uint8Array::from(value);
+            let mtime = JsValue::from_f64(mtime as f64);
+            cb.call3(&self.context, &key, &value, &mtime).map(|_| ())
+        };
+
+        let decompressed = flate2::read::GzDecoder::new(data);
+        let mut reader = tar::Archive::new(decompressed);
+        let entries = reader.entries();
+        let entries = entries.map_err(|err| {
+            let t = PackageError::MalformedArchive(Some(eco_format!("{err}")));
+            JsValue::from_str(&format!("{t:?}"))
+        })?;
+
+        let mut buf = Vec::with_capacity(1024);
+        for entry in entries {
+            // Read single entry
+            let mut entry = entry.map_err(|e| format!("{e:?}"))?;
+            let header = entry.header();
+
+            let is_file = header.entry_type().is_file();
+            if !is_file {
+                continue;
+            }
+
+            let mtime = header.mtime().unwrap_or(0);
+
+            let path = header.path().map_err(|e| format!("{e:?}"))?;
+            let path = path.to_string_lossy().as_ref().to_owned();
+
+            let size = header.size().map_err(|e| format!("{e:?}"))?;
+            buf.clear();
+            buf.reserve(size as usize);
+            entry.read_to_end(&mut buf).map_err(|e| format!("{e:?}"))?;
+
+            cb(path, &buf, mtime)?
+        }
+
+        Ok(())
+    }
+}
+
+/// The `JsRegistry` struct is a wrapper around a JavaScript function that
+#[derive(Debug)]
+pub struct JsRegistry {
+    /// The JavaScript this context.
+    pub context: ProxyContext,
+    /// The JavaScript function to call for resolving packages.
+    pub real_resolve_fn: js_sys::Function,
+}
+
+impl PackageRegistry for JsRegistry {
+    fn resolve(&self, spec: &PackageSpec) -> Result<std::sync::Arc<Path>, PackageError> {
+        // prepare js_spec
+        let js_spec = js_sys::Object::new();
+        js_sys::Reflect::set(&js_spec, &"name".into(), &spec.name.to_string().into()).unwrap();
+        js_sys::Reflect::set(
+            &js_spec,
+            &"namespace".into(),
+            &spec.namespace.to_string().into(),
+        )
+        .unwrap();
+        js_sys::Reflect::set(
+            &js_spec,
+            &"version".into(),
+            &spec.version.to_string().into(),
+        )
+        .unwrap();
+
+        self.real_resolve_fn
+            .call1(&self.context.clone().into(), &js_spec)
+            .map_err(|e| PackageError::Other(Some(eco_format!("{:?}", e))))
+            .and_then(|v| {
+                if v.is_undefined() {
+                    Err(PackageError::NotFound(spec.clone()))
+                } else {
+                    Ok(Path::new(&v.as_string().unwrap()).into())
+                }
+            })
+    }
+
+    // todo: provide package list for browser
+    fn packages(&self) -> &[(PackageSpec, Option<EcoString>)] {
+        &[]
+    }
+}
+
+// todo
+/// Safety: `JsRegistry` is only used in the browser environment, and we cannot
+/// share data between workers.
+unsafe impl Send for JsRegistry {}
+/// Safety: `JsRegistry` is only used in the browser environment, and we cannot
+/// share data between workers.
+unsafe impl Sync for JsRegistry {}

crates/tinymist-package/src/registry/dummy.rs

@@ -0,0 +1,15 @@
+//! Dummy package registry implementation for testing purposes.
+
+use std::{path::Path, sync::Arc};
+
+use super::{PackageError, PackageRegistry, PackageSpec};
+
+/// Dummy package registry that always returns a `NotFound` error.
+#[derive(Default, Debug)]
+pub struct DummyRegistry;
+
+impl PackageRegistry for DummyRegistry {
+    fn resolve(&self, spec: &PackageSpec) -> Result<Arc<Path>, PackageError> {
+        Err(PackageError::NotFound(spec.clone()))
+    }
+}

crates/tinymist-package/src/registry/http.rs

@@ -0,0 +1,336 @@
+//! Http registry for tinymist.
+
+use std::path::Path;
+use std::sync::{Arc, OnceLock};
+
+use parking_lot::Mutex;
+use reqwest::blocking::Response;
+use reqwest::Certificate;
+use tinymist_std::ImmutPath;
+use typst::diag::{eco_format, EcoString, PackageResult, StrResult};
+use typst::syntax::package::{PackageVersion, VersionlessPackageSpec};
+
+use super::{
+    DummyNotifier, Notifier, PackageError, PackageRegistry, PackageSpec, DEFAULT_REGISTRY,
+};
+
+/// The http package registry for typst.ts.
+pub struct HttpRegistry {
+    /// The path at which local packages (`@local` packages) are stored.
+    package_path: Option<ImmutPath>,
+    /// The path at which non-local packages (`@preview` packages) should be
+    /// stored when downloaded.
+    package_cache_path: Option<ImmutPath>,
+    /// lazily initialized package storage.
+    storage: OnceLock<PackageStorage>,
+    /// The path to the certificate file to use for HTTPS requests.
+    cert_path: Option<ImmutPath>,
+    /// The notifier to use for progress updates.
+    notifier: Arc<Mutex<dyn Notifier + Send>>,
+    // package_dir_cache: RwLock<HashMap<PackageSpec, Result<ImmutPath, PackageError>>>,
+}
+
+impl Default for HttpRegistry {
+    fn default() -> Self {
+        Self {
+            notifier: Arc::new(Mutex::<DummyNotifier>::default()),
+            cert_path: None,
+            package_path: None,
+            package_cache_path: None,
+
+            storage: OnceLock::new(),
+            // package_dir_cache: RwLock::new(HashMap::new()),
+        }
+    }
+}
+
+impl std::ops::Deref for HttpRegistry {
+    type Target = PackageStorage;
+
+    fn deref(&self) -> &Self::Target {
+        self.storage()
+    }
+}
+
+impl HttpRegistry {
+    /// Create a new registry.
+    pub fn new(
+        cert_path: Option<ImmutPath>,
+        package_path: Option<ImmutPath>,
+        package_cache_path: Option<ImmutPath>,
+    ) -> Self {
+        Self {
+            cert_path,
+            package_path,
+            package_cache_path,
+            ..Default::default()
+        }
+    }
+
+    /// Get `typst-kit` implementing package storage
+    pub fn storage(&self) -> &PackageStorage {
+        self.storage.get_or_init(|| {
+            PackageStorage::new(
+                self.package_cache_path
+                    .clone()
+                    .or_else(|| Some(dirs::cache_dir()?.join(DEFAULT_PACKAGES_SUBDIR).into())),
+                self.package_path
+                    .clone()
+                    .or_else(|| Some(dirs::data_dir()?.join(DEFAULT_PACKAGES_SUBDIR).into())),
+                self.cert_path.clone(),
+                self.notifier.clone(),
+            )
+        })
+    }
+
+    /// Get local path option
+    pub fn local_path(&self) -> Option<ImmutPath> {
+        self.storage().package_path().cloned()
+    }
+
+    /// Get data & cache dir
+    pub fn paths(&self) -> Vec<ImmutPath> {
+        let data_dir = self.storage().package_path().cloned();
+        let cache_dir = self.storage().package_cache_path().cloned();
+        data_dir.into_iter().chain(cache_dir).collect::<Vec<_>>()
+    }
+
+    /// Set list of packages for testing.
+    pub fn test_package_list(&self, f: impl FnOnce() -> Vec<(PackageSpec, Option<EcoString>)>) {
+        self.storage().index.get_or_init(f);
+    }
+}
+
+impl PackageRegistry for HttpRegistry {
+    fn resolve(&self, spec: &PackageSpec) -> Result<ImmutPath, PackageError> {
+        self.storage().prepare_package(spec)
+    }
+
+    fn packages(&self) -> &[(PackageSpec, Option<EcoString>)] {
+        self.storage().download_index()
+    }
+}
+
+/// The default packages sub directory within the package and package cache
+/// paths.
+pub const DEFAULT_PACKAGES_SUBDIR: &str = "typst/packages";
+
+/// Holds information about where packages should be stored and downloads them
+/// on demand, if possible.
+pub struct PackageStorage {
+    /// The path at which non-local packages should be stored when downloaded.
+    package_cache_path: Option<ImmutPath>,
+    /// The path at which local packages are stored.
+    package_path: Option<ImmutPath>,
+    /// The downloader used for fetching the index and packages.
+    cert_path: Option<ImmutPath>,
+    /// The cached index of the preview namespace.
+    index: OnceLock<Vec<(PackageSpec, Option<EcoString>)>>,
+    notifier: Arc<Mutex<dyn Notifier + Send>>,
+}
+
+impl PackageStorage {
+    /// Creates a new package storage for the given package paths.
+    /// It doesn't fallback directories, thus you can disable the related
+    /// storage by passing `None`.
+    pub fn new(
+        package_cache_path: Option<ImmutPath>,
+        package_path: Option<ImmutPath>,
+        cert_path: Option<ImmutPath>,
+        notifier: Arc<Mutex<dyn Notifier + Send>>,
+    ) -> Self {
+        Self {
+            package_cache_path,
+            package_path,
+            cert_path,
+            notifier,
+            index: OnceLock::new(),
+        }
+    }
+
+    /// Returns the path at which non-local packages should be stored when
+    /// downloaded.
+    pub fn package_cache_path(&self) -> Option<&ImmutPath> {
+        self.package_cache_path.as_ref()
+    }
+
+    /// Returns the path at which local packages are stored.
+    pub fn package_path(&self) -> Option<&ImmutPath> {
+        self.package_path.as_ref()
+    }
+
+    /// Make a package available in the on-disk cache.
+    pub fn prepare_package(&self, spec: &PackageSpec) -> PackageResult<ImmutPath> {
+        let subdir = format!("{}/{}/{}", spec.namespace, spec.name, spec.version);
+
+        if let Some(packages_dir) = &self.package_path {
+            let dir = packages_dir.join(&subdir);
+            if dir.exists() {
+                return Ok(dir.into());
+            }
+        }
+
+        if let Some(cache_dir) = &self.package_cache_path {
+            let dir = cache_dir.join(&subdir);
+            if dir.exists() {
+                return Ok(dir.into());
+            }
+
+            // Download from network if it doesn't exist yet.
+            if spec.namespace == "preview" {
+                self.download_package(spec, &dir)?;
+                if dir.exists() {
+                    return Ok(dir.into());
+                }
+            }
+        }
+
+        Err(PackageError::NotFound(spec.clone()))
+    }
+
+    /// Try to determine the latest version of a package.
+    pub fn determine_latest_version(
+        &self,
+        spec: &VersionlessPackageSpec,
+    ) -> StrResult<PackageVersion> {
+        if spec.namespace == "preview" {
+            // For `@preview`, download the package index and find the latest
+            // version.
+            self.download_index()
+                .iter()
+                .filter(|(package, _)| package.name == spec.name)
+                .map(|(package, _)| package.version)
+                .max()
+                .ok_or_else(|| eco_format!("failed to find package {spec}"))
+        } else {
+            // For other namespaces, search locally. We only search in the data
+            // directory and not the cache directory, because the latter is not
+            // intended for storage of local packages.
+            let subdir = format!("{}/{}", spec.namespace, spec.name);
+            self.package_path
+                .iter()
+                .flat_map(|dir| std::fs::read_dir(dir.join(&subdir)).ok())
+                .flatten()
+                .filter_map(|entry| entry.ok())
+                .map(|entry| entry.path())
+                .filter_map(|path| path.file_name()?.to_string_lossy().parse().ok())
+                .max()
+                .ok_or_else(|| eco_format!("please specify the desired version"))
+        }
+    }
+
+    /// Get the cached package index without network access.
+    pub fn cached_index(&self) -> Option<&[(PackageSpec, Option<EcoString>)]> {
+        self.index.get().map(Vec::as_slice)
+    }
+
+    /// Download the package index. The result of this is cached for efficiency.
+    pub fn download_index(&self) -> &[(PackageSpec, Option<EcoString>)] {
+        self.index.get_or_init(|| {
+            let url = format!("{DEFAULT_REGISTRY}/preview/index.json");
+
+            threaded_http(&url, self.cert_path.as_deref(), |resp| {
+                let reader = match resp.and_then(|r| r.error_for_status()) {
+                    Ok(response) => response,
+                    Err(err) => {
+                        // todo: silent error
+                        log::error!("Failed to fetch package index: {err} from {url}");
+                        return vec![];
+                    }
+                };
+
+                #[derive(serde::Deserialize)]
+                struct RemotePackageIndex {
+                    name: EcoString,
+                    version: PackageVersion,
+                    description: Option<EcoString>,
+                }
+
+                let indices: Vec<RemotePackageIndex> = match serde_json::from_reader(reader) {
+                    Ok(index) => index,
+                    Err(err) => {
+                        log::error!("Failed to parse package index: {err} from {url}");
+                        return vec![];
+                    }
+                };
+
+                indices
+                    .into_iter()
+                    .map(|index| {
+                        (
+                            PackageSpec {
+                                namespace: "preview".into(),
+                                name: index.name,
+                                version: index.version,
+                            },
+                            index.description,
+                        )
+                    })
+                    .collect::<Vec<_>>()
+            })
+            .unwrap_or_default()
+        })
+    }
+
+    /// Download a package over the network.
+    ///
+    /// # Panics
+    /// Panics if the package spec namespace isn't `preview`.
+    pub fn download_package(&self, spec: &PackageSpec, package_dir: &Path) -> PackageResult<()> {
+        assert_eq!(spec.namespace, "preview");
+
+        let url = format!(
+            "{DEFAULT_REGISTRY}/preview/{}-{}.tar.gz",
+            spec.name, spec.version
+        );
+
+        self.notifier.lock().downloading(spec);
+        threaded_http(&url, self.cert_path.as_deref(), |resp| {
+            let reader = match resp.and_then(|r| r.error_for_status()) {
+                Ok(response) => response,
+                Err(err) if matches!(err.status().map(|s| s.as_u16()), Some(404)) => {
+                    return Err(PackageError::NotFound(spec.clone()))
+                }
+                Err(err) => return Err(PackageError::NetworkFailed(Some(eco_format!("{err}")))),
+            };
+
+            let decompressed = flate2::read::GzDecoder::new(reader);
+            tar::Archive::new(decompressed)
+                .unpack(package_dir)
+                .map_err(|err| {
+                    std::fs::remove_dir_all(package_dir).ok();
+                    PackageError::MalformedArchive(Some(eco_format!("{err}")))
+                })
+        })
+        .ok_or_else(|| PackageError::Other(Some(eco_format!("cannot spawn http thread"))))?
+    }
+}
+
+pub(crate) fn threaded_http<T: Send + Sync>(
+    url: &str,
+    cert_path: Option<&Path>,
+    f: impl FnOnce(Result<Response, reqwest::Error>) -> T + Send + Sync,
+) -> Option<T> {
+    std::thread::scope(|s| {
+        s.spawn(move || {
+            let client_builder = reqwest::blocking::Client::builder();
+
+            let client = if let Some(cert_path) = cert_path {
+                let cert = std::fs::read(cert_path)
+                    .ok()
+                    .and_then(|buf| Certificate::from_pem(&buf).ok());
+                if let Some(cert) = cert {
+                    client_builder.add_root_certificate(cert).build().unwrap()
+                } else {
+                    client_builder.build().unwrap()
+                }
+            } else {
+                client_builder.build().unwrap()
+            };
+
+            f(client.get(url).send())
+        })
+        .join()
+        .ok()
+    })
+}