feat: local package manager (#458)

* feat: import local packages

* feat: create and open local packages

* dev: unwrap nested block

* dev: data directory as resource

---------

Co-authored-by: Myriad-Dreamin <camiyoru@gmail.com>

Cargo.lock

@@ -3788,6 +3788,7 @@ dependencies = [
  "comemo 0.4.0",
  "crossbeam-channel",
  "dhat",
+ "dirs",
  "env_logger",
  "futures",
  "hyper",
@@ -3852,6 +3853,7 @@ dependencies = [
  "chrono",
  "comemo 0.4.0",
  "dashmap",
+ "dirs",
  "ecow 0.2.2",
  "ena",
  "hashbrown 0.14.5",

Cargo.toml

@@ -41,6 +41,7 @@ open = { version = "5.1.3" }
 parking_lot = "0.12.1"
 walkdir = "2"
 chrono = "0.4"
+dirs = "5"
 
 # Networking
 hyper = { version = "0.14", features = ["full"] }

crates/tinymist-query/Cargo.toml

@@ -14,6 +14,7 @@ repository.workspace = true
 
 anyhow.workspace = true
 comemo.workspace = true
+dirs.workspace = true
 regex.workspace = true
 yaml-rust2.workspace = true
 biblatex.workspace = true

crates/tinymist-query/src/upstream/complete.rs

@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
 use typst::foundations::{fields_on, format_str, repr, Repr, StyleChain, Styles, Value};
 use typst::model::Document;
 use typst::syntax::ast::AstNode;
+use typst::syntax::package::PackageSpec;
 use typst::syntax::{ast, is_id_continue, is_id_start, is_ident, LinkedNode, Source, SyntaxKind};
 use typst::text::RawElem;
 use unscanny::Scanner;
@@ -1046,9 +1047,75 @@ impl<'a, 'w> CompletionContext<'a, 'w> {
         }
     }
 
+    /// Get local packages
+    fn local_packages(&mut self) -> Vec<(PackageSpec, Option<EcoString>)> {
+        // search packages locally. We only search in the data
+        // directory and not the cache directory, because the latter is not
+        // intended for storage of local packages.
+        let mut packages = vec![];
+        let Some(data_dir) = dirs::data_dir() else {
+            return packages;
+        };
+        let local_path = data_dir.join("typst/packages");
+        if !local_path.exists() {
+            return packages;
+        }
+        // namespace/package_name/version
+        // 1. namespace
+        let namespaces = std::fs::read_dir(local_path).unwrap();
+        for namespace in namespaces {
+            let namespace = namespace.unwrap();
+            if !namespace.file_type().unwrap().is_dir() {
+                continue;
+            }
+            // start with . are hidden directories
+            if namespace.file_name().to_string_lossy().starts_with('.') {
+                continue;
+            }
+            // 2. package_name
+            let package_names = std::fs::read_dir(namespace.path()).unwrap();
+            for package in package_names {
+                let package = package.unwrap();
+                if !package.file_type().unwrap().is_dir() {
+                    continue;
+                }
+                if package.file_name().to_string_lossy().starts_with('.') {
+                    continue;
+                }
+                // 3. version
+                let versions = std::fs::read_dir(package.path()).unwrap();
+                for version in versions {
+                    let version = version.unwrap();
+                    if !version.file_type().unwrap().is_dir() {
+                        continue;
+                    }
+                    if version.file_name().to_string_lossy().starts_with('.') {
+                        continue;
+                    }
+                    let version = version.file_name().to_string_lossy().parse().unwrap();
+                    let spec = PackageSpec {
+                        namespace: namespace.file_name().to_string_lossy().into(),
+                        name: package.file_name().to_string_lossy().into(),
+                        version,
+                    };
+                    let description = eco_format!("{} v{}", spec.name, spec.version);
+                    let package = (spec, Some(description));
+                    packages.push(package);
+                }
+            }
+        }
+        packages
+    }
+
     /// Add completions for all available packages.
     fn package_completions(&mut self, all_versions: bool) {
         let mut packages: Vec<_> = self.world().packages().iter().collect();
+        // local_packages to references and add them to the packages
+        let local_packages = self.local_packages();
+        let local_packages_refs: Vec<&(PackageSpec, Option<EcoString>)> =
+            local_packages.iter().collect();
+        packages.extend(local_packages_refs);
+
         packages.sort_by_key(|(spec, _)| (&spec.namespace, &spec.name, Reverse(spec.version)));
         if !all_versions {
             packages.dedup_by_key(|(spec, _)| (&spec.namespace, &spec.name));

crates/tinymist/Cargo.toml

@@ -66,6 +66,7 @@ dhat = { workspace = true, optional = true }
 unicode-script.workspace = true
 hyper = { workspace = true, optional = true }
 open = { workspace = true, optional = true }
+dirs.workspace = true
 tower-layer.workspace = true
 tower-service.workspace = true
 pin-project-lite.workspace = true

crates/tinymist/src/cmd.rs

@@ -491,4 +491,20 @@ impl LanguageState {
     pub fn resource_tutoral(&mut self, _arguments: Vec<JsonValue>) -> AnySchedulableResponse {
         Err(method_not_found())
     }
+
+    /// Get directory of local pacakges
+    pub fn resource_local_packages(
+        &mut self,
+        _arguments: Vec<JsonValue>,
+    ) -> AnySchedulableResponse {
+        let Some(data_dir) = dirs::data_dir() else {
+            return just_ok(JsonValue::Null);
+        };
+        let local_path = data_dir.join("typst/packages");
+        if !local_path.exists() {
+            return just_ok(JsonValue::Null);
+        }
+        let local_path = local_path.to_string_lossy().to_string();
+        just_ok(JsonValue::String(local_path))
+    }
 }

crates/tinymist/src/server.rs

@@ -260,7 +260,8 @@ impl LanguageState {
             // resources
             .with_resource("/symbols", State::resource_symbols)
             .with_resource("/preview/index.html", State::resource_preview_html)
-            .with_resource("/tutorial", State::resource_tutoral);
+            .with_resource("/tutorial", State::resource_tutoral)
+            .with_resource("/dirs/local-packages", State::resource_local_packages);
 
         // todo: generalize me
         provider.args.add_commands(

editors/vscode/package.json

@@ -828,6 +828,16 @@
         "title": "Show available Typst templates (gallery) for picking up a template to initialize",
         "category": "Typst"
       },
+      {
+        "command": "tinymist.createLocalPackage",
+        "title": "Create Typst Local Package",
+        "category": "Typst"
+      },
+      {
+        "command": "tinymist.openLocalPackage",
+        "title": "Open Typst Local Package",
+        "category": "Typst"
+      },
       {
         "command": "tinymist.showSummary",
         "title": "Show current document summary",

editors/vscode/src/extension.ts

@@ -31,6 +31,7 @@ import {
   previewPreload,
   previewProcessOutline,
 } from "./preview";
+import { commandCreateLocalPackage, commandOpenLocalPackage } from "./package-manager";
 import { activeTypstEditor, DisposeList, getSensibleTextEditorColumn } from "./util";
 import { client, getClient, setClient, tinymist } from "./lsp";
 import { taskActivate } from "./tasks";
@@ -262,6 +263,9 @@ async function startClient(client: LanguageClient, context: ExtensionContext): P
     commands.registerCommand("tinymist.showSymbolView", editorToolCommand("symbol-view")),
     commands.registerCommand("tinymist.profileCurrentFile", editorToolCommand("tracing")),
 
+    commands.registerCommand("tinymist.createLocalPackage", commandCreateLocalPackage),
+    commands.registerCommand("tinymist.openLocalPackage", commandOpenLocalPackage),
+
     // We would like to define it at the server side, but it is not possible for now.
     // https://github.com/microsoft/language-server-protocol/issues/1117
     commands.registerCommand("tinymist.triggerNamedCompletion", triggerNamedCompletion),

editors/vscode/src/lsp.ts

@@ -18,8 +18,9 @@ export async function getClient(): Promise<LanguageClient> {
 }
 
 interface ResourceRoutes {
-  "/symbols": any;
-  "/preview/index.html": string;
+    "/symbols": any;
+    "/preview/index.html": string;
+    "/dirs/local-packages": string;
 }
 
 export const tinymist = {

editors/vscode/src/package-manager.ts

@@ -0,0 +1,190 @@
+import { window, workspace } from 'vscode';
+import { tinymist } from './lsp';
+import * as fs from 'fs';
+
+// error message
+export const dataDirErrorMessage = 'Can not find package directory.';
+
+export async function getLocalPackagesDir() {
+    const packagesDir = await tinymist.getResource('/dirs/local-packages');
+    return packagesDir ? `${packagesDir}/local` : null;
+}
+
+// typst.toml template
+const typstTomlTemplate = (name: string, version: string, entrypoint: string) => {
+    return `[package]\nname = "${name}"\nversion = "${version}"\nentrypoint = "${entrypoint}"`;
+};
+
+// versionCompare
+function versionCompare(a: string, b: string) {
+    const aArr = a.split('.');
+    const bArr = b.split('.');
+    for (let i = 0; i < 3; i++) {
+        const aNum = Number(aArr[i]);
+        const bNum = Number(bArr[i]);
+        if (aNum !== bNum) {
+            return bNum - aNum;
+        }
+    }
+    return 0;
+}
+
+/**
+ * get local packages list
+ */
+export async function getLocalPackagesList() {
+    const localPackagesDir = await getLocalPackagesDir();
+    // return list of local packages like ['@local/mypkg:1.0.0']
+    if (!localPackagesDir) {
+        return [];
+    }
+    // if localPackagesDir doesn't exist, return []
+    try {
+        await fs.promises.access(localPackagesDir);
+    } catch (err) {
+        return [];
+    }
+    const localPackagesList = await fs.promises.readdir(localPackagesDir);
+    // get all version
+    const res = [] as {
+        package: string,
+        namespace: string,
+        name: string,
+        version: string,
+    }[];
+    for (const localPackage of localPackagesList) {
+        // if localPackage is not a directory, continue
+        const stat = await fs.promises.stat(`${localPackagesDir}/${localPackage}`);
+        if (!stat.isDirectory()) {
+            continue;
+        }
+        // filter versions only valid version like '0.1.0'
+        const versions = (await fs.promises.readdir(`${localPackagesDir}/${localPackage}`)).filter(version => {
+            const versionReg = /^\d+\.\d+\.\d+$/;
+            return versionReg.test(version);
+        });
+        // sort versions like ['1.0.0', '0.2.0', '0.1.0', '0.0.2', '0.0.1']
+        versions.sort(versionCompare);
+        for (const version of versions) {
+            res.push({
+                package: `@local/${localPackage}:${version}`,
+                namespace: 'local',
+                name: localPackage,
+                version,
+            });
+        }
+    }
+    return res;
+}
+
+/**
+ * create local package
+ */
+export async function commandCreateLocalPackage() {
+    const localPackagesDir = await getLocalPackagesDir();
+    if (!localPackagesDir) {
+        window.showErrorMessage(dataDirErrorMessage);
+        return;
+    }
+    // 1. input package name
+    const packageName = await window.showInputBox({
+        value: '',
+        placeHolder: 'Please input package name',
+        validateInput: text => {
+            return text ? null : 'Please input package name';
+        }
+    });
+    if (!packageName) {
+        return;
+    }
+    // 2. input package version
+    const packageVersion = await window.showInputBox({
+        value: '0.1.0',
+        placeHolder: 'Please input package version',
+        validateInput: text => {
+            if (!text) {
+                return 'Please input package version';
+            }
+            // make sure it is valid version like '0.1.0'
+            const versionReg = /^\d+\.\d+\.\d+$/;
+            if (!versionReg.test(text)) {
+                return 'Please input valid package version like 0.1.0';
+            }
+            return null;
+        }
+    });
+    if (!packageVersion) {
+        return;
+    }
+    // 3. input entrypoint
+    const entrypoint = await window.showInputBox({
+        value: 'lib.typ',
+        placeHolder: 'Please input entrypoint',
+        validateInput: text => {
+            if (!text) {
+                return 'Please input entrypoint';
+            }
+            // make sure it is valid entrypoint end with .typ
+            if (!text.endsWith('.typ')) {
+                return 'Please input valid entrypoint end with .typ';
+            }
+            return null;
+        }
+    });
+    if (!entrypoint) {
+        return;
+    }
+    // 4. create localPackagesDir/name/version/typst.toml
+    const packageDir = `${localPackagesDir}/${packageName}/${packageVersion}`;
+    const typstToml = typstTomlTemplate(packageName, packageVersion, entrypoint);
+    await fs.promises.mkdir(packageDir, { recursive: true });
+    await fs.promises.writeFile(`${packageDir}/typst.toml`, typstToml);
+    // 5. create localPackagesDir/name/version/entrypoint
+    await fs.promises.writeFile(`${packageDir}/${entrypoint}`, '#let add(a, b) = { a + b }');
+    // 6. open localPackagesDir/name/version/entrypoint
+    const document = await workspace.openTextDocument(`${packageDir}/${entrypoint}`);
+    await window.showTextDocument(document);
+}
+
+/**
+ * open local package in editor
+ */
+export async function commandOpenLocalPackage() {
+    const localPackagesDir = await getLocalPackagesDir();
+    if (!localPackagesDir) {
+        window.showErrorMessage(dataDirErrorMessage);
+        return;
+    }
+    // 1. select local package
+    const localPackagesList = await getLocalPackagesList();
+    const localPackages = localPackagesList.map(pkg => pkg.package);
+    const selected = await window.showQuickPick(localPackages, {
+        placeHolder: 'Please select a local package to open'
+    });
+    if (!selected) {
+        return;
+    }
+    // 2. read localPackagesDir/name/version/typst.toml
+    const name = localPackagesList.filter(pkg => pkg.package === selected)[0].name;
+    const version = localPackagesList.filter(pkg => pkg.package === selected)[0].version;
+    const packageDir = `${localPackagesDir}/${name}/${version}`;
+    // if typst.toml doesn't exist, return
+    try {
+        await fs.promises.access(`${packageDir}/typst.toml`);
+    } catch (err) {
+        window.showErrorMessage('Can not find typst.toml.');
+        return;
+    }
+    const typstToml = await fs.readFileSync(`${packageDir}/typst.toml`, 'utf-8');
+    // parse typst.toml
+    const entrypoint = typstToml.match(/entrypoint\s*=\s*"(.*)"/)?.[1];
+    if (!entrypoint) {
+        // open typst.toml if entrypoint is not set
+        const document = await workspace.openTextDocument(`${packageDir}/typst.toml`);
+        await window.showTextDocument(document);
+        return;
+    }
+    // 3. open localPackagesDir/name/version/entrypoint
+    const document = await workspace.openTextDocument(`${packageDir}/${entrypoint}`);
+    await window.showTextDocument(document);
+}