language-servers/tinymist
1
0
Fork 0
mirror of https://github.com/Myriad-Dreamin/tinymist.git synced 2025-07-07 21:15:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat: employ l10n to tinymist-cli and vscode extension (#1505)

Browse source 
* feat: runtime translation

* feat: poc of rust translation

* feat: clean up implementation

* feat: initialize correctly

* dev: remove dirty log

* dev: rename l10nMsg

* fix: desc

* feat: update assets building

* feat: update assets building

* build: update cargo.lock

* fix: warnings

* fix: warnings

* dev: expose api

* fix: compile error

* fix: compile errors in scripts
This commit is contained in:
Myriad-Dreamin 2025-03-15 10:38:07 +08:00 • committed by GitHub
parent dc9013e253
commit 4cbe35a286
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 615 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/release-asset-crate.yml vendored
View file

@ -35,9 +35,10 @@ jobs:
# uses: Swatinem/rust-cache@v2 # uses: Swatinem/rust-cache@v2
- name: Install deps - name: Install deps
run: yarn install run: yarn install
- name: Check and build typst-preview - name: Check and build assets
run: | run: |
yarn build:preview yarn build:preview
yarn build:l10n
- name: Publish crates - name: Publish crates
run: | run: |
cargo publish --allow-dirty --no-verify -p tinymist-assets || true cargo publish --allow-dirty --no-verify -p tinymist-assets || true

5
.github/workflows/release-vscode.yml vendored
View file

@ -52,10 +52,11 @@ jobs:
node-version: 22 node-version: 22
- name: Install deps - name: Install deps
run: yarn install run: yarn install
- name: Check and build typst-preview - uses: Swatinem/rust-cache@v2
- name: Check and build assets
run: | run: |
yarn build:preview yarn build:preview
- uses: Swatinem/rust-cache@v2 yarn build:l10n
- run: cargo clippy --workspace --all-targets - run: cargo clippy --workspace --all-targets
- run: cargo fmt --check --all - run: cargo fmt --check --all
- run: cargo doc --workspace --no-deps - run: cargo doc --workspace --no-deps

16
Cargo.lock generated
View file

@ -4027,6 +4027,7 @@ dependencies = [
"temp-env", "temp-env",
"tinymist-assets 0.13.8 (registry+https://github.com/rust-lang/crates.io-index)", "tinymist-assets 0.13.8 (registry+https://github.com/rust-lang/crates.io-index)",
"tinymist-core", "tinymist-core",
"tinymist-l10n",
"tinymist-project", "tinymist-project",
"tinymist-query", "tinymist-query",
"tinymist-render", "tinymist-render",
@ -4105,6 +4106,20 @@ dependencies = [
"syn 2.0.98", "syn 2.0.98",
] ]
[[package]]
name = "tinymist-l10n"
version = "0.13.8"
dependencies = [
"anyhow",
"clap",
"ecow",
"insta",
"rayon",
"rustc-hash 2.1.1",
"serde_json",
"walkdir",
]
[[package]] [[package]]
name = "tinymist-project" name = "tinymist-project"
version = "0.13.2" version = "0.13.2"
@ -4173,6 +4188,7 @@ dependencies = [
"strum", "strum",
"tinymist-analysis", "tinymist-analysis",
"tinymist-derive", "tinymist-derive",
"tinymist-l10n",
"tinymist-project", "tinymist-project",
"tinymist-std", "tinymist-std",
"tinymist-world", "tinymist-world",

1
Cargo.toml
View file

@ -187,6 +187,7 @@ typst-shim = { path = "./crates/typst-shim", version = "0.13.2" }
tinymist-core = { path = "./crates/tinymist-core/", version = "0.13.8", default-features = false } tinymist-core = { path = "./crates/tinymist-core/", version = "0.13.8", default-features = false }
tinymist = { path = "./crates/tinymist/", version = "0.13.8" } tinymist = { path = "./crates/tinymist/", version = "0.13.8" }
tinymist-l10n = { path = "./crates/tinymist-l10n/", version = "0.13.8" }
tinymist-query = { path = "./crates/tinymist-query/", version = "0.13.8" } tinymist-query = { path = "./crates/tinymist-query/", version = "0.13.8" }
tinymist-render = { path = "./crates/tinymist-render/", version = "0.13.8" } tinymist-render = { path = "./crates/tinymist-render/", version = "0.13.8" }
typst-preview = { path = "./crates/typst-preview", version = "0.13.8" } typst-preview = { path = "./crates/typst-preview", version = "0.13.8" }

1
crates/tinymist-assets/Cargo.toml
View file

@ -14,3 +14,4 @@ include = ["src/**/*"]
[features] [features]
typst-preview = [] typst-preview = []
l10n = []

3
crates/tinymist-assets/src/.gitignore vendored
View file

@ -1 +1,2 @@
typst-preview.html typst-preview.html
tinymist-rt.toml

7
crates/tinymist-assets/src/lib.rs
View file

@ -3,3 +3,10 @@
pub const TYPST_PREVIEW_HTML: &str = include_str!("typst-preview.html"); pub const TYPST_PREVIEW_HTML: &str = include_str!("typst-preview.html");
#[cfg(not(feature = "typst-preview"))] #[cfg(not(feature = "typst-preview"))]
pub const TYPST_PREVIEW_HTML: &str = "<html><body>Typst Preview needs to be built with the `embed-html` feature to work!</body></html>"; pub const TYPST_PREVIEW_HTML: &str = "<html><body>Typst Preview needs to be built with the `embed-html` feature to work!</body></html>";
/// If this file is not found, please runs `yarn extract:l10n:rs` to extract the
/// localization data.
#[cfg(feature = "l10n")]
pub const L10N_DATA: &str = include_str!("tinymist-rt.toml");
#[cfg(not(feature = "l10n"))]
pub const L10N_DATA: &str = "";

3
crates/tinymist-derive/Cargo.toml
View file

@ -18,6 +18,3 @@ quote.workspace = true
[lib] [lib]
proc-macro = true proc-macro = true
[features]
typst-preview = []

41
crates/tinymist-l10n/Cargo.toml Normal file
View file

@ -0,0 +1,41 @@
[package]
name = "tinymist-l10n"
description = "Localization support for tinymist and typst."
categories = ["compilers", "command-line-utilities"]
keywords = ["language", "typst"]
authors.workspace = true
version.workspace = true
license.workspace = true
edition.workspace = true
homepage.workspace = true
repository.workspace = true
rust-version.workspace = true
[[bin]]
name = "tinymist-l10n"
path = "src/main.rs"
required-features = ["cli"]
test = false
doctest = false
bench = false
doc = false
[dependencies]
anyhow.workspace = true
clap = { workspace = true, optional = true }
ecow.workspace = true
rayon.workspace = true
rustc-hash.workspace = true
serde_json.workspace = true
walkdir.workspace = true
[dev-dependencies]
insta.workspace = true
[features]
default = ["cli"]
cli = ["clap", "clap/wrap_help"]
[lints]
workspace = true

3
crates/tinymist-l10n/README.md Normal file
View file

@ -0,0 +1,3 @@
# tinymist-l10n
Tinymist's l10n tool.

3
crates/tinymist-l10n/dist.toml Normal file
View file

@ -0,0 +1,3 @@
[dist]
dist = false

251
crates/tinymist-l10n/src/lib.rs Normal file
View file

@ -0,0 +1,251 @@
//! Tinymist's localization library.
use core::panic;
use std::{
borrow::Cow,
collections::HashSet,
path::Path,
sync::{OnceLock, RwLock},
};
use rayon::{
iter::{IntoParallelRefMutIterator, ParallelIterator},
str::ParallelString,
};
use rustc_hash::FxHashMap;
/// A map of translations.
pub type TranslationMap = FxHashMap<String, String>;
/// A set of translation maps.
pub type TranslationMapSet = FxHashMap<String, TranslationMap>;
static ALL_TRANSLATIONS: OnceLock<TranslationMapSet> = OnceLock::new();
static LOCALE_TRANSLATIONS: RwLock<Option<&'static TranslationMap>> = RwLock::new(Option::None);
/// Sets the current translations. It can only be called once.
pub fn set_translations(translations: TranslationMapSet) {
let new_translations = ALL_TRANSLATIONS.set(translations);
if let Err(new_translations) = new_translations {
eprintln!("cannot set translations: len = {}", new_translations.len());
}
}
/// Sets the current locale.
pub fn set_locale(locale: &str) -> Option<()> {
let translations = ALL_TRANSLATIONS.get()?;
let lower_locale = locale.to_lowercase();
let locale = lower_locale.as_str();
let translations = translations.get(locale).or_else(|| {
// Tries s to find a language that starts with the locale and follow a hyphen.
translations
.iter()
.find(|(k, _)| locale.starts_with(*k) && locale.chars().nth(k.len()) == Some('-'))
.map(|(_, v)| v)
})?;
*LOCALE_TRANSLATIONS.write().unwrap() = Some(translations);
Some(())
}
/// Loads a TOML string into a map of translations.
pub fn load_translations(input: &str) -> anyhow::Result<TranslationMapSet> {
let mut translations = deserialize(input, false)?;
translations.par_iter_mut().for_each(|(_, v)| {
v.par_iter_mut().for_each(|(_, v)| {
if !v.starts_with('"') {
return;
}
*v = serde_json::from_str::<String>(v)
.unwrap_or_else(|e| panic!("cannot parse translation message: {e}, message: {v}"));
});
});
Ok(translations)
}
/// Updates disk translations with new key-value pairs.
pub fn update_disk_translations(
mut key_values: Vec<(String, String)>,
output: &Path,
) -> anyhow::Result<()> {
key_values.sort_by(|(key_x, _), (key_y, _)| key_x.cmp(key_y));
// Reads and parses existing translations
let mut translations = match std::fs::read_to_string(output) {
Ok(existing_translations) => deserialize(&existing_translations, true)?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => TranslationMapSet::default(),
Err(e) => Err(e)?,
};
// Removes unused translations
update_translations(key_values, &mut translations);
// Writes translations
let result = serialize_translations(translations);
std::fs::write(output, result)?;
Ok(())
}
/// Updates a map of translations with new key-value pairs.
pub fn update_translations(
key_values: Vec<(String, String)>,
translations: &mut TranslationMapSet,
) {
let used = key_values.iter().map(|e| &e.0).collect::<HashSet<_>>();
translations.retain(|k, _| used.contains(k));
// Updates translations
let en = "en".to_owned();
for (key, value) in key_values {
translations
.entry(key)
.or_default()
.insert(en.clone(), value);
}
}
/// Writes a map of translations to a TOML string.
pub fn serialize_translations(translations: TranslationMapSet) -> String {
let mut result = String::new();
result.push_str("\n# The translations are partially generated by copilot\n");
let mut translations = translations.into_iter().collect::<Vec<_>>();
translations.sort_by(|a, b| a.0.cmp(&b.0));
for (key, mut data) in translations {
result.push_str(&format!("\n[{key}]\n"));
let en = data.remove("en").expect("en translation is missing");
result.push_str(&format!("en = {en}\n"));
// sort by lang
let mut data = data.into_iter().collect::<Vec<_>>();
data.sort_by(|a, b| a.0.cmp(&b.0));
for (lang, value) in data {
result.push_str(&format!("{lang} = {value}\n"));
}
}
result
}
/// Tries to translate a string to the current language.
#[macro_export]
macro_rules! t {
($key:expr, $message:expr) => {
$crate::t_without_args($key, $message)
};
($key:expr, $message:expr, $($args:expr),*) => {
$crate::t_with_args($key, $message, &[$($args),*])
};
}
/// Tries to get a translation for a key.
fn find_message(key: &'static str, message: &'static str) -> &'static str {
let Some(translations) = LOCALE_TRANSLATIONS.read().unwrap().as_ref().copied() else {
return message;
};
translations.get(key).map(String::as_str).unwrap_or(message)
}
/// Tries to translate a string to the current language.
pub fn t_without_args(key: &'static str, message: &'static str) -> Cow<'static, str> {
Cow::Borrowed(find_message(key, message))
}
/// An argument for a translation.
pub enum Arg<'a> {
/// A string argument.
Str(&'a str),
/// An integer argument.
Int(i64),
/// A float argument.
Float(f64),
}
/// Tries to translate a string to the current language.
pub fn t_with_args(key: &'static str, message: &'static str, args: &[&Arg]) -> Cow<'static, str> {
let message = find_message(key, message);
let mut result = String::new();
let mut arg_index = 0;
for c in message.chars() {
if c == '{' {
let mut arg_index_str = String::new();
let chars = message.chars().skip(arg_index + 1);
for c in chars {
if c == '}' {
break;
}
arg_index_str.push(c);
}
arg_index = arg_index_str.parse::<usize>().unwrap();
let arg = args[arg_index];
match arg {
Arg::Str(s) => result.push_str(s),
Arg::Int(i) => result.push_str(&i.to_string()),
Arg::Float(f) => result.push_str(&f.to_string()),
}
arg_index += arg_index_str.len() + 2;
} else {
result.push(c);
}
}
Cow::Owned(result)
}
/// Deserializes a TOML string into a map of translations.
pub fn deserialize(input: &str, key_first: bool) -> anyhow::Result<TranslationMapSet> {
let lines = input
.par_split('\n')
.map(|line| line.trim())
.filter(|line| !line.starts_with('#') && !line.is_empty())
.collect::<Vec<_>>();
let mut translations = FxHashMap::default();
let mut key = String::new();
for line in lines {
if line.starts_with('[') {
key = line[1..line.len() - 1].to_string();
} else {
let equal_index = line.find('=').map_or_else(
|| {
Err(anyhow::anyhow!(
"cannot find equal sign in translation line: {line}"
))
},
Ok,
)?;
let lang = line[..equal_index].trim().to_string();
let value = line[equal_index + 1..].trim().to_string();
if key_first {
translations
.entry(key.clone())
.or_insert_with(FxHashMap::default)
.insert(lang, value);
} else {
translations
.entry(lang)
.or_insert_with(FxHashMap::default)
.insert(key.clone(), value);
}
}
}
Ok(translations)
}

125
crates/tinymist-l10n/src/main.rs Normal file
View file

@ -0,0 +1,125 @@
//! Fully parallelized l10n tool for Rust and TypeScript.
use std::path::Path;
use clap::Parser;
use rayon::{
iter::{ParallelBridge, ParallelIterator},
str::ParallelString,
};
use tinymist_l10n::update_disk_translations;
/// The CLI arguments of the tool.
#[derive(Debug, Clone, PartialEq, Eq, Parser)]
struct Args {
/// The kind of file to process.
///
/// It can be `rs` for Rust or `ts` for TypeScript.
/// - `rs`: checks `tinymist_l10n::t!` macro in Rust files.
/// - `ts`: checks `l10nMsg` function in TypeScript files.
#[clap(long)]
kind: String,
/// The directory to process recursively.
#[clap(long)]
dir: String,
/// The output file to write the translations. The file will be in-place
/// updated with new translations.
#[clap(long)]
output: String,
}
fn main() -> anyhow::Result<()> {
let args = Args::parse();
let is_rs = args.kind == "rs";
let file_calls = walkdir::WalkDir::new(&args.dir)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.filter(|e| {
e.path()
.extension()
.is_some_and(|e| e == args.kind.as_str())
})
.par_bridge()
.flat_map(|e| check_calls(e, is_rs))
.collect::<Vec<_>>();
update_disk_translations(file_calls, Path::new(&args.output))?;
Ok(())
}
const L10N_FN_TS: &str = "l10nMsg";
const L10N_FN_RS: &str = "tinymist_l10n::t!";
fn check_calls(e: walkdir::DirEntry, is_rs: bool) -> Vec<(String, String)> {
let path = e.path();
let content = match std::fs::read_to_string(path) {
Ok(content) => content,
Err(err) => {
eprintln!("failed to read file {path:?}: {err}");
return Vec::new();
}
};
content
.as_str()
.par_match_indices(if is_rs { 't' } else { 'l' })
.flat_map(|e| {
let s = &content[e.0..];
if !is_rs && s.starts_with(L10N_FN_TS) {
let suffix = &content[e.0 + L10N_FN_TS.len()..];
return parse_l10n_args_ts(suffix);
fn parse_l10n_args_ts(s: &str) -> Option<(String, String)> {
let s = parse_char(s, '(')?;
let (key, _s) = parse_str(s)?;
Some((format!("\"{key}\""), format!("\"{key}\"")))
}
}
if is_rs && s.starts_with(L10N_FN_RS) {
let suffix = &content[e.0 + L10N_FN_RS.len()..];
return parse_l10n_args_rs(suffix);
fn parse_l10n_args_rs(s: &str) -> Option<(String, String)> {
let s = parse_char(s, '(')?;
let (key, s) = parse_str(s)?;
let s = parse_char(s, ',')?;
let (value, _s) = parse_str(s)?;
Some((key.to_string(), format!("\"{value}\"")))
}
}
None
})
.collect::<Vec<_>>()
}
fn parse_char(s: &str, ch: char) -> Option<&str> {
let s = s.trim_start();
if s.starts_with(ch) {
Some(&s[1..])
} else {
None
}
}
fn parse_str(s: &str) -> Option<(&str, &str)> {
let s = parse_char(s, '"')?;
let mut escape = false;
for (i, ch) in s.char_indices() {
if escape {
escape = false;
} else {
match ch {
'\\' => escape = true,
'"' => return Some((&s[..i], &s[i + 1..])),
_ => (),
}
}
}
None
}

1
crates/tinymist-query/Cargo.toml
View file

@ -60,6 +60,7 @@ tinymist-project = { workspace = true, features = ["lsp"] }
tinymist-analysis.workspace = true tinymist-analysis.workspace = true
tinymist-derive.workspace = true tinymist-derive.workspace = true
tinymist-std.workspace = true tinymist-std.workspace = true
tinymist-l10n.workspace = true
[dev-dependencies] [dev-dependencies]
once_cell.workspace = true once_cell.workspace = true

25
crates/tinymist-query/src/code_lens.rs
View file

@ -31,18 +31,33 @@ impl SemanticRequest for CodeLensRequest {
data: None, data: None,
}; };
res.push(doc_lens("Profile", vec!["profile".into()])); res.push(doc_lens(
res.push(doc_lens("Preview", vec!["preview".into()])); &tinymist_l10n::t!("tinymist-query.code-action.profile", "Profile"),
vec!["profile".into()],
));
res.push(doc_lens(
&tinymist_l10n::t!("tinymist-query.code-action.preview", "Preview"),
vec!["preview".into()],
));
let is_html = ctx.world.library.features.is_enabled(typst::Feature::Html); let is_html = ctx.world.library.features.is_enabled(typst::Feature::Html);
if is_html { if is_html {
res.push(doc_lens("Export HTML", vec!["export-html".into()])); res.push(doc_lens(
&tinymist_l10n::t!("tinymist-query.code-action.exportHtml", "Export HTML"),
vec!["export-html".into()],
));
} else { } else {
res.push(doc_lens("Export PDF", vec!["export-pdf".into()])); res.push(doc_lens(
&tinymist_l10n::t!("tinymist-query.code-action.exportPdf", "Export PDF"),
vec!["export-pdf".into()],
));
} }
res.push(doc_lens("More ..", vec!["more".into()])); res.push(doc_lens(
&tinymist_l10n::t!("tinymist-query.code-action.more", "More .."),
vec!["more".into()],
));
Some(res) Some(res)
} }

15
crates/tinymist/Cargo.toml
View file

@ -83,9 +83,19 @@ typstfmt.workspace = true
typstyle-core.workspace = true typstyle-core.workspace = true
unicode-script.workspace = true unicode-script.workspace = true
walkdir.workspace = true walkdir.workspace = true
tinymist-l10n.workspace = true
[features] [features]
default = ["cli", "html", "pdf", "preview", "embed-fonts", "no-content-hint"] default = [
"cli",
"html",
"pdf",
# enable it once we upload assets.
# "l10n",
"preview",
"embed-fonts",
"no-content-hint",
]
cli = ["sync-lsp/clap", "clap/wrap_help"] cli = ["sync-lsp/clap", "clap/wrap_help"]
@ -121,6 +131,9 @@ preview = [
"hyper-tungstenite", "hyper-tungstenite",
] ]
# l10n = ["tinymist-assets/l10n"]
l10n = []
[dev-dependencies] [dev-dependencies]
temp-env.workspace = true temp-env.workspace = true

13
crates/tinymist/src/init.rs
View file

@ -73,6 +73,10 @@ impl Initializer for RegularInit {
fn initialize(self, params: InitializeParams) -> (ServerState, AnySchedulableResponse) { fn initialize(self, params: InitializeParams) -> (ServerState, AnySchedulableResponse) {
let (config, err) = Config::from_params(params, self.font_opts); let (config, err) = Config::from_params(params, self.font_opts);
if let Some(locale) = config.const_config.locale.as_ref() {
tinymist_l10n::set_locale(locale);
}
let super_init = SuperInit { let super_init = SuperInit {
client: self.client, client: self.client,
exec_cmds: self.exec_cmds, exec_cmds: self.exec_cmds,
@ -524,6 +528,8 @@ pub struct ConstConfig {
pub doc_line_folding_only: bool, pub doc_line_folding_only: bool,
/// Allow dynamic registration of document formatting. /// Allow dynamic registration of document formatting.
pub doc_fmt_dynamic_registration: bool, pub doc_fmt_dynamic_registration: bool,
/// The locale of the editor.
pub locale: Option<String>,
} }
impl Default for ConstConfig { impl Default for ConstConfig {
@ -555,6 +561,12 @@ impl From<&InitializeParams> for ConstConfig {
let fold = try_(|| doc?.folding_range.as_ref()); let fold = try_(|| doc?.folding_range.as_ref());
let format = try_(|| doc?.formatting.as_ref()); let format = try_(|| doc?.formatting.as_ref());
let locale = params
.initialization_options
.as_ref()
.and_then(|init| init.get("locale").and_then(|v| v.as_str()))
.or(params.locale.as_deref());
Self { Self {
position_encoding, position_encoding,
cfg_change_registration: try_or(|| workspace?.configuration, false), cfg_change_registration: try_or(|| workspace?.configuration, false),
@ -564,6 +576,7 @@ impl From<&InitializeParams> for ConstConfig {
tokens_multiline_token_support: try_or(|| sema?.multiline_token_support, false), tokens_multiline_token_support: try_or(|| sema?.multiline_token_support, false),
doc_line_folding_only: try_or(|| fold?.line_folding_only, true), doc_line_folding_only: try_or(|| fold?.line_folding_only, true),
doc_fmt_dynamic_registration: try_or(|| format?.dynamic_registration, false), doc_fmt_dynamic_registration: try_or(|| format?.dynamic_registration, false),
locale: locale.map(ToOwned::to_owned),
} }
} }
} }

17
crates/tinymist/src/main.rs
View file

@ -26,6 +26,9 @@ use tinymist_project::EntryResolver;
use tinymist_query::package::PackageInfo; use tinymist_query::package::PackageInfo;
use tinymist_std::{bail, error::prelude::*}; use tinymist_std::{bail, error::prelude::*};
#[cfg(feature = "l10n")]
use tinymist_l10n::{load_translations, set_translations};
use crate::args::*; use crate::args::*;
#[cfg(feature = "dhat-heap")] #[cfg(feature = "dhat-heap")]
@ -56,13 +59,21 @@ fn main() -> Result<()> {
#[cfg(feature = "dhat-heap")] #[cfg(feature = "dhat-heap")]
let _profiler = dhat::Profiler::new_heap(); let _profiler = dhat::Profiler::new_heap();
// Parse command line arguments // Parses command line arguments
let args = CliArguments::parse(); let args = CliArguments::parse();
let is_transient_cmd = matches!(args.command, Some(Commands::Compile(..))); // Probes soon to avoid other initializations causing errors
if matches!(args.command, Some(Commands::Probe)) {
return Ok(());
}
// Start logging // Loads translations
#[cfg(feature = "l10n")]
set_translations(load_translations(tinymist_assets::L10N_DATA)?);
// Starts logging
let _ = { let _ = {
let is_transient_cmd = matches!(args.command, Some(Commands::Compile(..)));
use log::LevelFilter::*; use log::LevelFilter::*;
let base_level = if is_transient_cmd { Warn } else { Info }; let base_level = if is_transient_cmd { Warn } else { Info };

4
docs/tinymist/configurations.typ
View file

@ -25,9 +25,9 @@
// todo: anyOf // todo: anyOf
} else if cfg.type == "array" [ } else if cfg.type == "array" [
- Items: #raw(cfg.items.type) - Items: #raw(cfg.items.type)
- Description: #md(cfg.items.description) - Description: #md(cfg.items.at("markdownDescription", default: cfg.at("description", default: "")))
] ]
- Description: #md(cfg.at("markdownDescription", default: cfg.at("description", default: none))) - Description: #md(cfg.at("markdownDescription", default: cfg.at("description", default: "")))
#if cfg.at("enum", default: none) != none [ #if cfg.at("enum", default: none) != none [
- Valid values: #for (i, item) in cfg.enum.enumerate() [ - Valid values: #for (i, item) in cfg.enum.enumerate() [
- #raw(item): #if "enumDescriptions" in cfg { md(cfg.enumDescriptions.at(i)) } - #raw(item): #if "enumDescriptions" in cfg { md(cfg.enumDescriptions.at(i)) }

3
editors/vscode/.gitignore vendored
View file

@ -6,4 +6,5 @@ coverage
icons/ti.svg icons/ti.svg
icons/typst-small.svg icons/typst-small.svg
package.nls.json package.nls.json
package.nls.*.json package.nls.*.json
l10n/**/*

1
editors/vscode/.vscodeignore
View file

@ -1,4 +1,5 @@
** **
!l10n/**/*
!package.nls.json !package.nls.json
!package.nls.*.json !package.nls.*.json
!out/tinymist-docs.pdf !out/tinymist-docs.pdf

2
editors/vscode/Configuration.md
View file

@ -165,7 +165,7 @@ Sets the indent size (using space) for the formatter.
## `tinymist.showExportFileIn` ## `tinymist.showExportFileIn`
(Experimental) Show Exported Files in Some Place Configures way of opening exported files, e.g. inside of editor tabs or using system application.
## `tinymist.dragAndDrop` ## `tinymist.dragAndDrop`

1
editors/vscode/package.json
View file

@ -29,6 +29,7 @@
"main": "./out/extension.js", "main": "./out/extension.js",
"browser": "./out/extension.web.js", "browser": "./out/extension.web.js",
"icon": "./icons/ti-white.png", "icon": "./icons/ti-white.png",
"l10n": "./l10n",
"contributes": { "contributes": {
"viewsContainers": { "viewsContainers": {
"activitybar": [ "activitybar": [

10
editors/vscode/scripts/config-man.mjs
View file

@ -3,8 +3,7 @@ import * as path from "path";
import { vscodeExtTranslations } from "../../../scripts/build-l10n.mjs"; import { vscodeExtTranslations } from "../../../scripts/build-l10n.mjs";
const __dirname = new URL(".", import.meta.url).toString().replace("file:///", ""); const projectRoot = path.join(import.meta.dirname, "../../..");
const projectRoot = path.join(__dirname, "../../..");
const packageJsonPath = path.join(projectRoot, "editors/vscode/package.json"); const packageJsonPath = path.join(projectRoot, "editors/vscode/package.json");
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
@ -155,7 +154,7 @@ ${typeSection}${enumSection}${defaultSection}
.join("\n"); .join("\n");
}; };
const configMdPath = path.join(__dirname, "..", "Configuration.md"); const configMdPath = path.join(import.meta.dirname, "..", "Configuration.md");
fs.writeFileSync( fs.writeFileSync(
configMdPath, configMdPath,
@ -164,7 +163,10 @@ fs.writeFileSync(
${configMd("vscode", true)}`, ${configMd("vscode", true)}`,
); );
const configMdPathNeovim = path.join(__dirname, "../../../editors/neovim/Configuration.md"); const configMdPathNeovim = path.join(
import.meta.dirname,
"../../../editors/neovim/Configuration.md",
);
fs.writeFileSync( fs.writeFileSync(
configMdPathNeovim, configMdPathNeovim,

3
editors/vscode/src/features/preview.ts
View file

@ -24,6 +24,7 @@ import {
ScrollPreviewRequest, ScrollPreviewRequest,
tinymist, tinymist,
} from "../lsp"; } from "../lsp";
import { l10nMsg } from "../l10n";
/** /**
* The launch preview implementation which depends on `isCompat` of previewActivate. * The launch preview implementation which depends on `isCompat` of previewActivate.
@ -259,7 +260,7 @@ export async function openPreviewInWebView({
? webviewPanel ? webviewPanel
: vscode.window.createWebviewPanel( : vscode.window.createWebviewPanel(
"typst-preview", "typst-preview",
`${basename} (Preview)`, `${basename}${l10nMsg(" (Preview)")}`,
getTargetViewColumn(activeEditor.viewColumn), getTargetViewColumn(activeEditor.viewColumn),
{ {
enableScripts: true, enableScripts: true,

5
editors/vscode/src/l10n.ts Normal file
View file

@ -0,0 +1,5 @@
import * as vscode from "vscode";
export function l10nMsg(message: string, args?: Record<string, string | number | boolean>): string {
return vscode.l10n.t(message, args || {});
}

4
eslint.config.mjs
View file

@ -2,12 +2,10 @@ import globals from "globals";
import typescriptEslint from "@typescript-eslint/eslint-plugin"; import typescriptEslint from "@typescript-eslint/eslint-plugin";
import tsParser from "@typescript-eslint/parser"; import tsParser from "@typescript-eslint/parser";
import path from "node:path"; import path from "node:path";
import { fileURLToPath } from "node:url";
import js from "@eslint/js"; import js from "@eslint/js";
import { FlatCompat } from "@eslint/eslintrc"; import { FlatCompat } from "@eslint/eslintrc";
const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(import.meta.dirname);
const __dirname = path.dirname(__filename);
const compat = new FlatCompat({ const compat = new FlatCompat({
baseDirectory: __dirname, baseDirectory: __dirname,
recommendedConfig: js.configs.recommended, recommendedConfig: js.configs.recommended,

22
locales/tinymist-rt.toml Normal file
View file

@ -0,0 +1,22 @@
# The translation are partially generated by copilot
[tinymist-query.code-action.exportHtml]
en = "Export HTML"
zh = "导出 HTML"
[tinymist-query.code-action.exportPdf]
en = "Export PDF"
zh = "导出 PDF"
[tinymist-query.code-action.more]
en = "More .."
zh = "更多 .."
[tinymist-query.code-action.preview]
en = "Preview"
zh = "预览"
[tinymist-query.code-action.profile]
en = "Profile"
zh = "性能分析"

6
locales/tinymist-vscode-rt.toml Normal file
View file

@ -0,0 +1,6 @@
# The translation are partially generated by copilot
[" (Preview)"]
en = " (Preview)"
zh = "（预览）"

5
package.json
View file

@ -16,7 +16,10 @@
"scripts": { "scripts": {
"build:editor-tools": "cd tools/editor-tools/ && yarn run build", "build:editor-tools": "cd tools/editor-tools/ && yarn run build",
"build:preview": "cd tools/typst-preview-frontend && yarn run build && rimraf ../../crates/tinymist-assets/src/typst-preview.html && cpr ./dist/index.html ../../crates/tinymist-assets/src/typst-preview.html", "build:preview": "cd tools/typst-preview-frontend && yarn run build && rimraf ../../crates/tinymist-assets/src/typst-preview.html && cpr ./dist/index.html ../../crates/tinymist-assets/src/typst-preview.html",
"build:l10n": "node scripts/build-l10n.mjs", "build:l10n": "yarn extract:l10n && node scripts/build-l10n.mjs",
"extract:l10n": "yarn extract:l10n:ts && yarn extract:l10n:rs",
"extract:l10n:ts": "cargo run --release --bin tinymist-l10n -- --kind ts --dir ./editors/vscode --output ./locales/tinymist-vscode-rt.toml",
"extract:l10n:rs": "cargo run --release --bin tinymist-l10n -- --kind rs --dir ./crates --output ./locales/tinymist-rt.toml && rimraf ./crates/tinymist-assets/src/tinymist-rt.toml && cpr ./locales/tinymist-rt.toml ./crates/tinymist-assets/src/tinymist-rt.toml",
"maintainers": "typst query MAINTAINERS.typ \"<maintainer-meta>\" --pretty --one --field value --input=action=help", "maintainers": "typst query MAINTAINERS.typ \"<maintainer-meta>\" --pretty --one --field value --input=action=help",
"docs": "shiroa serve --font-path assets/fonts -w . docs/tinymist", "docs": "shiroa serve --font-path assets/fonts -w . docs/tinymist",
"docs:pdf": "cargo run --bin tinymist --release -- compile --font-path assets/fonts --root . docs/tinymist/ebook.typ", "docs:pdf": "cargo run --bin tinymist --release -- compile --font-path assets/fonts --root . docs/tinymist/ebook.typ",

47
scripts/build-l10n.mjs
View file

@ -8,21 +8,31 @@
import * as fs from "fs"; import * as fs from "fs";
import * as path from "path"; import * as path from "path";
const __dirname = new URL(".", import.meta.url).toString().replace("file:///", ""); const projectRoot = path.resolve(import.meta.dirname, "..");
const projectRoot = path.resolve(__dirname, "..");
function translate(input, output) { /**
const data = fs.readFileSync(path.resolve(projectRoot, input), "utf-8"); *
const lines = data * @param {string} output
.split("\n") * @param {string[]} inputs
.map((line) => line.trim()) * @returns
.filter((line) => !line.startsWith("#") && line.length > 0); */
function translate(output, kind, inputs) {
const lines = inputs.flatMap((input) =>
fs
.readFileSync(path.resolve(projectRoot, input), "utf-8")
.split("\n")
.map((line) => line.trim())
.filter((line) => !line.startsWith("#") && line.length > 0),
);
const translations = {}; const translations = {};
let key = ""; let key = "";
for (let line of lines) { for (let line of lines) {
if (line.startsWith("[")) { if (line.startsWith("[")) {
key = line.substring(1, line.length - 1); key = line.substring(1, line.length - 1);
if (key.startsWith('"')) {
key = JSON.parse(key);
}
} else { } else {
const equalIndex = line.indexOf("="); const equalIndex = line.indexOf("=");
const lang = line.substring(0, equalIndex).trim(); const lang = line.substring(0, equalIndex).trim();
@ -42,21 +52,34 @@ function translate(input, output) {
const langRest = langs.filter((lang) => lang !== "en"); const langRest = langs.filter((lang) => lang !== "en");
const langEnPath = path.resolve(projectRoot, `${output}/package.nls.json`); const langDir = path.resolve(projectRoot, output);
fs.mkdirSync(langDir, { recursive: true });
const langEnPath = `${langDir}/${kind}.json`;
const langEnData = translations["en"]; const langEnData = translations["en"];
fs.writeFileSync(langEnPath, JSON.stringify(langEnData, null, 2)); fs.writeFileSync(langEnPath, JSON.stringify(langEnData, null, 2));
for (let lang of langRest) { for (let lang of langRest) {
const langPath = path.resolve(projectRoot, `${output}/package.nls.${lang}.json`); const langPath = `${langDir}/${kind}.${lang}.json`;
const langData = translations[lang]; const langData = translations[lang];
fs.writeFileSync(langPath, JSON.stringify(langData, null, 2)); const langPack = JSON.stringify(langData, null, 2);
fs.writeFileSync(langPath, langPack);
// alias zh-cn
if (kind === "bundle.l10n" && lang === "zh") {
const langZhCNPath = `${langDir}/${kind}.zh-cn.json`;
fs.writeFileSync(langZhCNPath, langPack);
}
} }
return translations; return translations;
} }
// todo: verify using rust
function genVscodeExt() { function genVscodeExt() {
const translations = translate("locales/tinymist-vscode.toml", "editors/vscode"); const translations = translate("editors/vscode", "package.nls", ["locales/tinymist-vscode.toml"]);
translate("editors/vscode/l10n", "bundle.l10n", ["locales/tinymist-vscode-rt.toml"]);
const pat = /\%(extension\.tinymist\..*?)\%/g; const pat = /\%(extension\.tinymist\..*?)\%/g;
const data = fs.readFileSync(path.resolve(projectRoot, "editors/vscode/package.json"), "utf-8"); const data = fs.readFileSync(path.resolve(projectRoot, "editors/vscode/package.json"), "utf-8");

24
scripts/link-docs.mjs
View file

@ -1,10 +1,8 @@
import { resolve, basename } from "path"; import { resolve, basename } from "path";
import { fileURLToPath } from "url";
import * as fs from "fs"; import * as fs from "fs";
import { execSync } from "child_process"; import { execSync } from "child_process";
const filename = fileURLToPath(import.meta.url); const root = resolve(import.meta.dirname, "../..");
const root = resolve(filename, "../..");
const dry = process.argv.includes("--dry"); const dry = process.argv.includes("--dry");
const bytes2utf8 = new TextDecoder(); const bytes2utf8 = new TextDecoder();
@ -59,7 +57,7 @@ const convert = async (inp, out, opts) => {
imageCnt += 1; imageCnt += 1;
fs.writeFileSync(resolve(assetsDir, fileName), base64Decode(content)); fs.writeFileSync(resolve(assetsDir, fileName), base64Decode(content));
return `"./assets/images/${fileName}"`; return `"./assets/images/${fileName}"`;
} },
); );
await fs.promises.writeFile(output, outputContent); await fs.promises.writeFile(output, outputContent);
@ -70,13 +68,9 @@ const main = async () => {
convert("docs/tinymist/introduction.typ", "README.md", { convert("docs/tinymist/introduction.typ", "README.md", {
before: "# Tinymist\n\n", before: "# Tinymist\n\n",
}), }),
convert( convert("docs/tinymist/release-instruction.typ", "docs/release-instruction.md", {
"docs/tinymist/release-instruction.typ", before: "# Release Instructions\n\n",
"docs/release-instruction.md", }),
{
before: "# Release Instructions\n\n",
}
),
convert("docs/tinymist/frontend/emacs.typ", "editors/emacs/README.md", { convert("docs/tinymist/frontend/emacs.typ", "editors/emacs/README.md", {
before: "# Tinymist Emacs Support for Typst\n\n", before: "# Tinymist Emacs Support for Typst\n\n",
}), }),
@ -86,11 +80,9 @@ const main = async () => {
convert("docs/tinymist/frontend/neovim.typ", "editors/neovim/README.md", { convert("docs/tinymist/frontend/neovim.typ", "editors/neovim/README.md", {
before: "# Tinymist Neovim Support for Typst\n\n", before: "# Tinymist Neovim Support for Typst\n\n",
}), }),
convert( convert("docs/tinymist/frontend/sublime-text.typ", "editors/sublime-text/README.md", {
"docs/tinymist/frontend/sublime-text.typ", before: "# Tinymist Sublime Support for Typst\n\n",
"editors/sublime-text/README.md", }),
{ before: "# Tinymist Sublime Support for Typst\n\n" }
),
convert("docs/tinymist/frontend/vscode.typ", "editors/vscode/README.md", { convert("docs/tinymist/frontend/vscode.typ", "editors/vscode/README.md", {
before: "# Tinymist Typst VS Code Extension\n\n", before: "# Tinymist Typst VS Code Extension\n\n",
}), }),

7
syntaxes/textmate/main.mts
View file

@ -14,7 +14,6 @@ import { blockRaw, blockRawGeneral, blockRawLangs, inlineRaw } from "./fenced.mj
import * as fs from "fs"; import * as fs from "fs";
import * as path from "path"; import * as path from "path";
import { fileURLToPath } from "node:url";
import { import {
FIXED_LENGTH_LOOK_BEHIND, FIXED_LENGTH_LOOK_BEHIND,
POLYFILL_P_XID, POLYFILL_P_XID,
@ -1465,8 +1464,6 @@ export const typst: textmate.Grammar = {
}; };
function generate() { function generate() {
const dirname = fileURLToPath(new URL(".", import.meta.url));
let compiled = textmate.compile(typst); let compiled = textmate.compile(typst);
if (POLYFILL_P_XID) { if (POLYFILL_P_XID) {
@ -1492,7 +1489,7 @@ function generate() {
// dump to file // dump to file
fs.writeFileSync( fs.writeFileSync(
path.join(dirname, "../typst.tmLanguage.json"), path.join(import.meta.dirname, "../typst.tmLanguage.json"),
JSON.stringify({ JSON.stringify({
$schema: "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", $schema: "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
scopeName: "source.typst", scopeName: "source.typst",
@ -1504,7 +1501,7 @@ function generate() {
// dump to file // dump to file
fs.writeFileSync( fs.writeFileSync(
path.join(dirname, "../typst-code.tmLanguage.json"), path.join(import.meta.dirname, "../typst-code.tmLanguage.json"),
JSON.stringify({ JSON.stringify({
$schema: "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", $schema: "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json",
scopeName: "source.typst-code", scopeName: "source.typst-code",