From 4cbe35a28662fa9d7a3b080b4800358113d6456c Mon Sep 17 00:00:00 2001 From: Myriad-Dreamin <35292584+Myriad-Dreamin@users.noreply.github.com> Date: Sat, 15 Mar 2025 10:38:07 +0800 Subject: [PATCH] feat: employ l10n to tinymist-cli and vscode extension (#1505) * feat: runtime translation * feat: poc of rust translation * feat: clean up implementation * feat: initialize correctly * dev: remove dirty log * dev: rename l10nMsg * fix: desc * feat: update assets building * feat: update assets building * build: update cargo.lock * fix: warnings * fix: warnings * dev: expose api * fix: compile error * fix: compile errors in scripts --- .github/workflows/release-asset-crate.yml | 3 +- .github/workflows/release-vscode.yml | 5 +- Cargo.lock | 16 ++ Cargo.toml | 1 + crates/tinymist-assets/Cargo.toml | 1 + crates/tinymist-assets/src/.gitignore | 3 +- crates/tinymist-assets/src/lib.rs | 7 + crates/tinymist-derive/Cargo.toml | 3 - crates/tinymist-l10n/Cargo.toml | 41 ++++ crates/tinymist-l10n/README.md | 3 + crates/tinymist-l10n/dist.toml | 3 + crates/tinymist-l10n/src/lib.rs | 251 ++++++++++++++++++++++ crates/tinymist-l10n/src/main.rs | 125 +++++++++++ crates/tinymist-query/Cargo.toml | 1 + crates/tinymist-query/src/code_lens.rs | 25 ++- crates/tinymist/Cargo.toml | 15 +- crates/tinymist/src/init.rs | 13 ++ crates/tinymist/src/main.rs | 17 +- docs/tinymist/configurations.typ | 4 +- editors/vscode/.gitignore | 3 +- editors/vscode/.vscodeignore | 1 + editors/vscode/Configuration.md | 2 +- editors/vscode/package.json | 1 + editors/vscode/scripts/config-man.mjs | 10 +- editors/vscode/src/features/preview.ts | 3 +- editors/vscode/src/l10n.ts | 5 + eslint.config.mjs | 4 +- locales/tinymist-rt.toml | 22 ++ locales/tinymist-vscode-rt.toml | 6 + package.json | 5 +- scripts/build-l10n.mjs | 47 ++-- scripts/link-docs.mjs | 24 +-- syntaxes/textmate/main.mts | 7 +- 33 files changed, 615 insertions(+), 62 deletions(-) create mode 100644 crates/tinymist-l10n/Cargo.toml create mode 100644 crates/tinymist-l10n/README.md create mode 100644 crates/tinymist-l10n/dist.toml create mode 100644 crates/tinymist-l10n/src/lib.rs create mode 100644 crates/tinymist-l10n/src/main.rs create mode 100644 editors/vscode/src/l10n.ts create mode 100644 locales/tinymist-rt.toml create mode 100644 locales/tinymist-vscode-rt.toml diff --git a/.github/workflows/release-asset-crate.yml b/.github/workflows/release-asset-crate.yml index 9fc3946a..d44b89bb 100644 --- a/.github/workflows/release-asset-crate.yml +++ b/.github/workflows/release-asset-crate.yml @@ -35,9 +35,10 @@ jobs: # uses: Swatinem/rust-cache@v2 - name: Install deps run: yarn install - - name: Check and build typst-preview + - name: Check and build assets run: | yarn build:preview + yarn build:l10n - name: Publish crates run: | cargo publish --allow-dirty --no-verify -p tinymist-assets || true diff --git a/.github/workflows/release-vscode.yml b/.github/workflows/release-vscode.yml index ca7e5da5..2f6ca576 100644 --- a/.github/workflows/release-vscode.yml +++ b/.github/workflows/release-vscode.yml @@ -52,10 +52,11 @@ jobs: node-version: 22 - name: Install deps run: yarn install - - name: Check and build typst-preview + - uses: Swatinem/rust-cache@v2 + - name: Check and build assets run: | yarn build:preview - - uses: Swatinem/rust-cache@v2 + yarn build:l10n - run: cargo clippy --workspace --all-targets - run: cargo fmt --check --all - run: cargo doc --workspace --no-deps diff --git a/Cargo.lock b/Cargo.lock index 44d3f130..446b5b09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4027,6 +4027,7 @@ dependencies = [ "temp-env", "tinymist-assets 0.13.8 (registry+https://github.com/rust-lang/crates.io-index)", "tinymist-core", + "tinymist-l10n", "tinymist-project", "tinymist-query", "tinymist-render", @@ -4105,6 +4106,20 @@ dependencies = [ "syn 2.0.98", ] +[[package]] +name = "tinymist-l10n" +version = "0.13.8" +dependencies = [ + "anyhow", + "clap", + "ecow", + "insta", + "rayon", + "rustc-hash 2.1.1", + "serde_json", + "walkdir", +] + [[package]] name = "tinymist-project" version = "0.13.2" @@ -4173,6 +4188,7 @@ dependencies = [ "strum", "tinymist-analysis", "tinymist-derive", + "tinymist-l10n", "tinymist-project", "tinymist-std", "tinymist-world", diff --git a/Cargo.toml b/Cargo.toml index 67c89381..a67fdb5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -187,6 +187,7 @@ typst-shim = { path = "./crates/typst-shim", version = "0.13.2" } tinymist-core = { path = "./crates/tinymist-core/", version = "0.13.8", default-features = false } tinymist = { path = "./crates/tinymist/", version = "0.13.8" } +tinymist-l10n = { path = "./crates/tinymist-l10n/", version = "0.13.8" } tinymist-query = { path = "./crates/tinymist-query/", version = "0.13.8" } tinymist-render = { path = "./crates/tinymist-render/", version = "0.13.8" } typst-preview = { path = "./crates/typst-preview", version = "0.13.8" } diff --git a/crates/tinymist-assets/Cargo.toml b/crates/tinymist-assets/Cargo.toml index 203809c1..96f2f08e 100644 --- a/crates/tinymist-assets/Cargo.toml +++ b/crates/tinymist-assets/Cargo.toml @@ -14,3 +14,4 @@ include = ["src/**/*"] [features] typst-preview = [] +l10n = [] diff --git a/crates/tinymist-assets/src/.gitignore b/crates/tinymist-assets/src/.gitignore index ad8e3db7..187b6e17 100644 --- a/crates/tinymist-assets/src/.gitignore +++ b/crates/tinymist-assets/src/.gitignore @@ -1 +1,2 @@ -typst-preview.html \ No newline at end of file +typst-preview.html +tinymist-rt.toml \ No newline at end of file diff --git a/crates/tinymist-assets/src/lib.rs b/crates/tinymist-assets/src/lib.rs index e07d824c..5c96011e 100644 --- a/crates/tinymist-assets/src/lib.rs +++ b/crates/tinymist-assets/src/lib.rs @@ -3,3 +3,10 @@ pub const TYPST_PREVIEW_HTML: &str = include_str!("typst-preview.html"); #[cfg(not(feature = "typst-preview"))] pub const TYPST_PREVIEW_HTML: &str = "Typst Preview needs to be built with the `embed-html` feature to work!"; + +/// If this file is not found, please runs `yarn extract:l10n:rs` to extract the +/// localization data. +#[cfg(feature = "l10n")] +pub const L10N_DATA: &str = include_str!("tinymist-rt.toml"); +#[cfg(not(feature = "l10n"))] +pub const L10N_DATA: &str = ""; diff --git a/crates/tinymist-derive/Cargo.toml b/crates/tinymist-derive/Cargo.toml index 1d1a650c..16d2597f 100644 --- a/crates/tinymist-derive/Cargo.toml +++ b/crates/tinymist-derive/Cargo.toml @@ -18,6 +18,3 @@ quote.workspace = true [lib] proc-macro = true - -[features] -typst-preview = [] diff --git a/crates/tinymist-l10n/Cargo.toml b/crates/tinymist-l10n/Cargo.toml new file mode 100644 index 00000000..7ee65d07 --- /dev/null +++ b/crates/tinymist-l10n/Cargo.toml @@ -0,0 +1,41 @@ +[package] +name = "tinymist-l10n" +description = "Localization support for tinymist and typst." +categories = ["compilers", "command-line-utilities"] +keywords = ["language", "typst"] +authors.workspace = true +version.workspace = true +license.workspace = true +edition.workspace = true +homepage.workspace = true +repository.workspace = true +rust-version.workspace = true + +[[bin]] +name = "tinymist-l10n" +path = "src/main.rs" +required-features = ["cli"] +test = false +doctest = false +bench = false +doc = false + +[dependencies] + +anyhow.workspace = true +clap = { workspace = true, optional = true } +ecow.workspace = true +rayon.workspace = true +rustc-hash.workspace = true +serde_json.workspace = true +walkdir.workspace = true + +[dev-dependencies] +insta.workspace = true + +[features] +default = ["cli"] +cli = ["clap", "clap/wrap_help"] + +[lints] +workspace = true diff --git a/crates/tinymist-l10n/README.md b/crates/tinymist-l10n/README.md new file mode 100644 index 00000000..a00474c2 --- /dev/null +++ b/crates/tinymist-l10n/README.md @@ -0,0 +1,3 @@ +# tinymist-l10n + +Tinymist's l10n tool. diff --git a/crates/tinymist-l10n/dist.toml b/crates/tinymist-l10n/dist.toml new file mode 100644 index 00000000..228c1309 --- /dev/null +++ b/crates/tinymist-l10n/dist.toml @@ -0,0 +1,3 @@ + +[dist] +dist = false diff --git a/crates/tinymist-l10n/src/lib.rs b/crates/tinymist-l10n/src/lib.rs new file mode 100644 index 00000000..eba0014a --- /dev/null +++ b/crates/tinymist-l10n/src/lib.rs @@ -0,0 +1,251 @@ +//! Tinymist's localization library. + +use core::panic; +use std::{ + borrow::Cow, + collections::HashSet, + path::Path, + sync::{OnceLock, RwLock}, +}; + +use rayon::{ + iter::{IntoParallelRefMutIterator, ParallelIterator}, + str::ParallelString, +}; +use rustc_hash::FxHashMap; + +/// A map of translations. +pub type TranslationMap = FxHashMap; +/// A set of translation maps. +pub type TranslationMapSet = FxHashMap; + +static ALL_TRANSLATIONS: OnceLock = OnceLock::new(); +static LOCALE_TRANSLATIONS: RwLock> = RwLock::new(Option::None); + +/// Sets the current translations. It can only be called once. +pub fn set_translations(translations: TranslationMapSet) { + let new_translations = ALL_TRANSLATIONS.set(translations); + + if let Err(new_translations) = new_translations { + eprintln!("cannot set translations: len = {}", new_translations.len()); + } +} + +/// Sets the current locale. +pub fn set_locale(locale: &str) -> Option<()> { + let translations = ALL_TRANSLATIONS.get()?; + let lower_locale = locale.to_lowercase(); + let locale = lower_locale.as_str(); + let translations = translations.get(locale).or_else(|| { + // Tries s to find a language that starts with the locale and follow a hyphen. + translations + .iter() + .find(|(k, _)| locale.starts_with(*k) && locale.chars().nth(k.len()) == Some('-')) + .map(|(_, v)| v) + })?; + + *LOCALE_TRANSLATIONS.write().unwrap() = Some(translations); + + Some(()) +} + +/// Loads a TOML string into a map of translations. +pub fn load_translations(input: &str) -> anyhow::Result { + let mut translations = deserialize(input, false)?; + translations.par_iter_mut().for_each(|(_, v)| { + v.par_iter_mut().for_each(|(_, v)| { + if !v.starts_with('"') { + return; + } + + *v = serde_json::from_str::(v) + .unwrap_or_else(|e| panic!("cannot parse translation message: {e}, message: {v}")); + }); + }); + + Ok(translations) +} + +/// Updates disk translations with new key-value pairs. +pub fn update_disk_translations( + mut key_values: Vec<(String, String)>, + output: &Path, +) -> anyhow::Result<()> { + key_values.sort_by(|(key_x, _), (key_y, _)| key_x.cmp(key_y)); + + // Reads and parses existing translations + let mut translations = match std::fs::read_to_string(output) { + Ok(existing_translations) => deserialize(&existing_translations, true)?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => TranslationMapSet::default(), + Err(e) => Err(e)?, + }; + + // Removes unused translations + update_translations(key_values, &mut translations); + + // Writes translations + let result = serialize_translations(translations); + std::fs::write(output, result)?; + Ok(()) +} + +/// Updates a map of translations with new key-value pairs. +pub fn update_translations( + key_values: Vec<(String, String)>, + translations: &mut TranslationMapSet, +) { + let used = key_values.iter().map(|e| &e.0).collect::>(); + translations.retain(|k, _| used.contains(k)); + + // Updates translations + let en = "en".to_owned(); + for (key, value) in key_values { + translations + .entry(key) + .or_default() + .insert(en.clone(), value); + } +} + +/// Writes a map of translations to a TOML string. +pub fn serialize_translations(translations: TranslationMapSet) -> String { + let mut result = String::new(); + + result.push_str("\n# The translations are partially generated by copilot\n"); + + let mut translations = translations.into_iter().collect::>(); + translations.sort_by(|a, b| a.0.cmp(&b.0)); + + for (key, mut data) in translations { + result.push_str(&format!("\n[{key}]\n")); + + let en = data.remove("en").expect("en translation is missing"); + result.push_str(&format!("en = {en}\n")); + + // sort by lang + let mut data = data.into_iter().collect::>(); + data.sort_by(|a, b| a.0.cmp(&b.0)); + + for (lang, value) in data { + result.push_str(&format!("{lang} = {value}\n")); + } + } + + result +} + +/// Tries to translate a string to the current language. +#[macro_export] +macro_rules! t { + ($key:expr, $message:expr) => { + $crate::t_without_args($key, $message) + }; + ($key:expr, $message:expr, $($args:expr),*) => { + $crate::t_with_args($key, $message, &[$($args),*]) + }; +} + +/// Tries to get a translation for a key. +fn find_message(key: &'static str, message: &'static str) -> &'static str { + let Some(translations) = LOCALE_TRANSLATIONS.read().unwrap().as_ref().copied() else { + return message; + }; + + translations.get(key).map(String::as_str).unwrap_or(message) +} + +/// Tries to translate a string to the current language. +pub fn t_without_args(key: &'static str, message: &'static str) -> Cow<'static, str> { + Cow::Borrowed(find_message(key, message)) +} + +/// An argument for a translation. +pub enum Arg<'a> { + /// A string argument. + Str(&'a str), + /// An integer argument. + Int(i64), + /// A float argument. + Float(f64), +} + +/// Tries to translate a string to the current language. +pub fn t_with_args(key: &'static str, message: &'static str, args: &[&Arg]) -> Cow<'static, str> { + let message = find_message(key, message); + let mut result = String::new(); + let mut arg_index = 0; + + for c in message.chars() { + if c == '{' { + let mut arg_index_str = String::new(); + + let chars = message.chars().skip(arg_index + 1); + + for c in chars { + if c == '}' { + break; + } + + arg_index_str.push(c); + } + + arg_index = arg_index_str.parse::().unwrap(); + let arg = args[arg_index]; + + match arg { + Arg::Str(s) => result.push_str(s), + Arg::Int(i) => result.push_str(&i.to_string()), + Arg::Float(f) => result.push_str(&f.to_string()), + } + + arg_index += arg_index_str.len() + 2; + } else { + result.push(c); + } + } + + Cow::Owned(result) +} + +/// Deserializes a TOML string into a map of translations. +pub fn deserialize(input: &str, key_first: bool) -> anyhow::Result { + let lines = input + .par_split('\n') + .map(|line| line.trim()) + .filter(|line| !line.starts_with('#') && !line.is_empty()) + .collect::>(); + + let mut translations = FxHashMap::default(); + let mut key = String::new(); + + for line in lines { + if line.starts_with('[') { + key = line[1..line.len() - 1].to_string(); + } else { + let equal_index = line.find('=').map_or_else( + || { + Err(anyhow::anyhow!( + "cannot find equal sign in translation line: {line}" + )) + }, + Ok, + )?; + let lang = line[..equal_index].trim().to_string(); + let value = line[equal_index + 1..].trim().to_string(); + + if key_first { + translations + .entry(key.clone()) + .or_insert_with(FxHashMap::default) + .insert(lang, value); + } else { + translations + .entry(lang) + .or_insert_with(FxHashMap::default) + .insert(key.clone(), value); + } + } + } + + Ok(translations) +} diff --git a/crates/tinymist-l10n/src/main.rs b/crates/tinymist-l10n/src/main.rs new file mode 100644 index 00000000..10f37deb --- /dev/null +++ b/crates/tinymist-l10n/src/main.rs @@ -0,0 +1,125 @@ +//! Fully parallelized l10n tool for Rust and TypeScript. + +use std::path::Path; + +use clap::Parser; +use rayon::{ + iter::{ParallelBridge, ParallelIterator}, + str::ParallelString, +}; +use tinymist_l10n::update_disk_translations; + +/// The CLI arguments of the tool. +#[derive(Debug, Clone, PartialEq, Eq, Parser)] +struct Args { + /// The kind of file to process. + /// + /// It can be `rs` for Rust or `ts` for TypeScript. + /// - `rs`: checks `tinymist_l10n::t!` macro in Rust files. + /// - `ts`: checks `l10nMsg` function in TypeScript files. + #[clap(long)] + kind: String, + /// The directory to process recursively. + #[clap(long)] + dir: String, + /// The output file to write the translations. The file will be in-place + /// updated with new translations. + #[clap(long)] + output: String, +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + + let is_rs = args.kind == "rs"; + let file_calls = walkdir::WalkDir::new(&args.dir) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .filter(|e| { + e.path() + .extension() + .is_some_and(|e| e == args.kind.as_str()) + }) + .par_bridge() + .flat_map(|e| check_calls(e, is_rs)) + .collect::>(); + + update_disk_translations(file_calls, Path::new(&args.output))?; + + Ok(()) +} + +const L10N_FN_TS: &str = "l10nMsg"; +const L10N_FN_RS: &str = "tinymist_l10n::t!"; + +fn check_calls(e: walkdir::DirEntry, is_rs: bool) -> Vec<(String, String)> { + let path = e.path(); + let content = match std::fs::read_to_string(path) { + Ok(content) => content, + Err(err) => { + eprintln!("failed to read file {path:?}: {err}"); + return Vec::new(); + } + }; + + content + .as_str() + .par_match_indices(if is_rs { 't' } else { 'l' }) + .flat_map(|e| { + let s = &content[e.0..]; + if !is_rs && s.starts_with(L10N_FN_TS) { + let suffix = &content[e.0 + L10N_FN_TS.len()..]; + return parse_l10n_args_ts(suffix); + + fn parse_l10n_args_ts(s: &str) -> Option<(String, String)> { + let s = parse_char(s, '(')?; + let (key, _s) = parse_str(s)?; + Some((format!("\"{key}\""), format!("\"{key}\""))) + } + } + if is_rs && s.starts_with(L10N_FN_RS) { + let suffix = &content[e.0 + L10N_FN_RS.len()..]; + return parse_l10n_args_rs(suffix); + + fn parse_l10n_args_rs(s: &str) -> Option<(String, String)> { + let s = parse_char(s, '(')?; + let (key, s) = parse_str(s)?; + let s = parse_char(s, ',')?; + let (value, _s) = parse_str(s)?; + Some((key.to_string(), format!("\"{value}\""))) + } + } + None + }) + .collect::>() +} + +fn parse_char(s: &str, ch: char) -> Option<&str> { + let s = s.trim_start(); + if s.starts_with(ch) { + Some(&s[1..]) + } else { + None + } +} + +fn parse_str(s: &str) -> Option<(&str, &str)> { + let s = parse_char(s, '"')?; + + let mut escape = false; + + for (i, ch) in s.char_indices() { + if escape { + escape = false; + } else { + match ch { + '\\' => escape = true, + '"' => return Some((&s[..i], &s[i + 1..])), + _ => (), + } + } + } + + None +} diff --git a/crates/tinymist-query/Cargo.toml b/crates/tinymist-query/Cargo.toml index 29eceec5..3b23e1b7 100644 --- a/crates/tinymist-query/Cargo.toml +++ b/crates/tinymist-query/Cargo.toml @@ -60,6 +60,7 @@ tinymist-project = { workspace = true, features = ["lsp"] } tinymist-analysis.workspace = true tinymist-derive.workspace = true tinymist-std.workspace = true +tinymist-l10n.workspace = true [dev-dependencies] once_cell.workspace = true diff --git a/crates/tinymist-query/src/code_lens.rs b/crates/tinymist-query/src/code_lens.rs index 01ca0f4c..ada07749 100644 --- a/crates/tinymist-query/src/code_lens.rs +++ b/crates/tinymist-query/src/code_lens.rs @@ -31,18 +31,33 @@ impl SemanticRequest for CodeLensRequest { data: None, }; - res.push(doc_lens("Profile", vec!["profile".into()])); - res.push(doc_lens("Preview", vec!["preview".into()])); + res.push(doc_lens( + &tinymist_l10n::t!("tinymist-query.code-action.profile", "Profile"), + vec!["profile".into()], + )); + res.push(doc_lens( + &tinymist_l10n::t!("tinymist-query.code-action.preview", "Preview"), + vec!["preview".into()], + )); let is_html = ctx.world.library.features.is_enabled(typst::Feature::Html); if is_html { - res.push(doc_lens("Export HTML", vec!["export-html".into()])); + res.push(doc_lens( + &tinymist_l10n::t!("tinymist-query.code-action.exportHtml", "Export HTML"), + vec!["export-html".into()], + )); } else { - res.push(doc_lens("Export PDF", vec!["export-pdf".into()])); + res.push(doc_lens( + &tinymist_l10n::t!("tinymist-query.code-action.exportPdf", "Export PDF"), + vec!["export-pdf".into()], + )); } - res.push(doc_lens("More ..", vec!["more".into()])); + res.push(doc_lens( + &tinymist_l10n::t!("tinymist-query.code-action.more", "More .."), + vec!["more".into()], + )); Some(res) } diff --git a/crates/tinymist/Cargo.toml b/crates/tinymist/Cargo.toml index 0a4fd2c0..b0f93028 100644 --- a/crates/tinymist/Cargo.toml +++ b/crates/tinymist/Cargo.toml @@ -83,9 +83,19 @@ typstfmt.workspace = true typstyle-core.workspace = true unicode-script.workspace = true walkdir.workspace = true +tinymist-l10n.workspace = true [features] -default = ["cli", "html", "pdf", "preview", "embed-fonts", "no-content-hint"] +default = [ + "cli", + "html", + "pdf", + # enable it once we upload assets. + # "l10n", + "preview", + "embed-fonts", + "no-content-hint", +] cli = ["sync-lsp/clap", "clap/wrap_help"] @@ -121,6 +131,9 @@ preview = [ "hyper-tungstenite", ] +# l10n = ["tinymist-assets/l10n"] +l10n = [] + [dev-dependencies] temp-env.workspace = true diff --git a/crates/tinymist/src/init.rs b/crates/tinymist/src/init.rs index 2be1d271..971a3a19 100644 --- a/crates/tinymist/src/init.rs +++ b/crates/tinymist/src/init.rs @@ -73,6 +73,10 @@ impl Initializer for RegularInit { fn initialize(self, params: InitializeParams) -> (ServerState, AnySchedulableResponse) { let (config, err) = Config::from_params(params, self.font_opts); + if let Some(locale) = config.const_config.locale.as_ref() { + tinymist_l10n::set_locale(locale); + } + let super_init = SuperInit { client: self.client, exec_cmds: self.exec_cmds, @@ -524,6 +528,8 @@ pub struct ConstConfig { pub doc_line_folding_only: bool, /// Allow dynamic registration of document formatting. pub doc_fmt_dynamic_registration: bool, + /// The locale of the editor. + pub locale: Option, } impl Default for ConstConfig { @@ -555,6 +561,12 @@ impl From<&InitializeParams> for ConstConfig { let fold = try_(|| doc?.folding_range.as_ref()); let format = try_(|| doc?.formatting.as_ref()); + let locale = params + .initialization_options + .as_ref() + .and_then(|init| init.get("locale").and_then(|v| v.as_str())) + .or(params.locale.as_deref()); + Self { position_encoding, cfg_change_registration: try_or(|| workspace?.configuration, false), @@ -564,6 +576,7 @@ impl From<&InitializeParams> for ConstConfig { tokens_multiline_token_support: try_or(|| sema?.multiline_token_support, false), doc_line_folding_only: try_or(|| fold?.line_folding_only, true), doc_fmt_dynamic_registration: try_or(|| format?.dynamic_registration, false), + locale: locale.map(ToOwned::to_owned), } } } diff --git a/crates/tinymist/src/main.rs b/crates/tinymist/src/main.rs index 430a854e..f1ee930f 100644 --- a/crates/tinymist/src/main.rs +++ b/crates/tinymist/src/main.rs @@ -26,6 +26,9 @@ use tinymist_project::EntryResolver; use tinymist_query::package::PackageInfo; use tinymist_std::{bail, error::prelude::*}; +#[cfg(feature = "l10n")] +use tinymist_l10n::{load_translations, set_translations}; + use crate::args::*; #[cfg(feature = "dhat-heap")] @@ -56,13 +59,21 @@ fn main() -> Result<()> { #[cfg(feature = "dhat-heap")] let _profiler = dhat::Profiler::new_heap(); - // Parse command line arguments + // Parses command line arguments let args = CliArguments::parse(); - let is_transient_cmd = matches!(args.command, Some(Commands::Compile(..))); + // Probes soon to avoid other initializations causing errors + if matches!(args.command, Some(Commands::Probe)) { + return Ok(()); + } - // Start logging + // Loads translations + #[cfg(feature = "l10n")] + set_translations(load_translations(tinymist_assets::L10N_DATA)?); + + // Starts logging let _ = { + let is_transient_cmd = matches!(args.command, Some(Commands::Compile(..))); use log::LevelFilter::*; let base_level = if is_transient_cmd { Warn } else { Info }; diff --git a/docs/tinymist/configurations.typ b/docs/tinymist/configurations.typ index a52e0c28..be9768c7 100644 --- a/docs/tinymist/configurations.typ +++ b/docs/tinymist/configurations.typ @@ -25,9 +25,9 @@ // todo: anyOf } else if cfg.type == "array" [ - Items: #raw(cfg.items.type) - - Description: #md(cfg.items.description) + - Description: #md(cfg.items.at("markdownDescription", default: cfg.at("description", default: ""))) ] - - Description: #md(cfg.at("markdownDescription", default: cfg.at("description", default: none))) + - Description: #md(cfg.at("markdownDescription", default: cfg.at("description", default: ""))) #if cfg.at("enum", default: none) != none [ - Valid values: #for (i, item) in cfg.enum.enumerate() [ - #raw(item): #if "enumDescriptions" in cfg { md(cfg.enumDescriptions.at(i)) } diff --git a/editors/vscode/.gitignore b/editors/vscode/.gitignore index 1a97004d..85ebd51b 100644 --- a/editors/vscode/.gitignore +++ b/editors/vscode/.gitignore @@ -6,4 +6,5 @@ coverage icons/ti.svg icons/typst-small.svg package.nls.json -package.nls.*.json \ No newline at end of file +package.nls.*.json +l10n/**/* \ No newline at end of file diff --git a/editors/vscode/.vscodeignore b/editors/vscode/.vscodeignore index 92f662da..ac79a75c 100644 --- a/editors/vscode/.vscodeignore +++ b/editors/vscode/.vscodeignore @@ -1,4 +1,5 @@ ** +!l10n/**/* !package.nls.json !package.nls.*.json !out/tinymist-docs.pdf diff --git a/editors/vscode/Configuration.md b/editors/vscode/Configuration.md index 0dc40a2a..b4d69fe8 100644 --- a/editors/vscode/Configuration.md +++ b/editors/vscode/Configuration.md @@ -165,7 +165,7 @@ Sets the indent size (using space) for the formatter. ## `tinymist.showExportFileIn` -(Experimental) Show Exported Files in Some Place +Configures way of opening exported files, e.g. inside of editor tabs or using system application. ## `tinymist.dragAndDrop` diff --git a/editors/vscode/package.json b/editors/vscode/package.json index 01a5d4bc..71966f70 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -29,6 +29,7 @@ "main": "./out/extension.js", "browser": "./out/extension.web.js", "icon": "./icons/ti-white.png", + "l10n": "./l10n", "contributes": { "viewsContainers": { "activitybar": [ diff --git a/editors/vscode/scripts/config-man.mjs b/editors/vscode/scripts/config-man.mjs index 1d563560..247fe63a 100644 --- a/editors/vscode/scripts/config-man.mjs +++ b/editors/vscode/scripts/config-man.mjs @@ -3,8 +3,7 @@ import * as path from "path"; import { vscodeExtTranslations } from "../../../scripts/build-l10n.mjs"; -const __dirname = new URL(".", import.meta.url).toString().replace("file:///", ""); -const projectRoot = path.join(__dirname, "../../.."); +const projectRoot = path.join(import.meta.dirname, "../../.."); const packageJsonPath = path.join(projectRoot, "editors/vscode/package.json"); const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); @@ -155,7 +154,7 @@ ${typeSection}${enumSection}${defaultSection} .join("\n"); }; -const configMdPath = path.join(__dirname, "..", "Configuration.md"); +const configMdPath = path.join(import.meta.dirname, "..", "Configuration.md"); fs.writeFileSync( configMdPath, @@ -164,7 +163,10 @@ fs.writeFileSync( ${configMd("vscode", true)}`, ); -const configMdPathNeovim = path.join(__dirname, "../../../editors/neovim/Configuration.md"); +const configMdPathNeovim = path.join( + import.meta.dirname, + "../../../editors/neovim/Configuration.md", +); fs.writeFileSync( configMdPathNeovim, diff --git a/editors/vscode/src/features/preview.ts b/editors/vscode/src/features/preview.ts index 6636dddd..688bb4d3 100644 --- a/editors/vscode/src/features/preview.ts +++ b/editors/vscode/src/features/preview.ts @@ -24,6 +24,7 @@ import { ScrollPreviewRequest, tinymist, } from "../lsp"; +import { l10nMsg } from "../l10n"; /** * The launch preview implementation which depends on `isCompat` of previewActivate. @@ -259,7 +260,7 @@ export async function openPreviewInWebView({ ? webviewPanel : vscode.window.createWebviewPanel( "typst-preview", - `${basename} (Preview)`, + `${basename}${l10nMsg(" (Preview)")}`, getTargetViewColumn(activeEditor.viewColumn), { enableScripts: true, diff --git a/editors/vscode/src/l10n.ts b/editors/vscode/src/l10n.ts new file mode 100644 index 00000000..52cf0410 --- /dev/null +++ b/editors/vscode/src/l10n.ts @@ -0,0 +1,5 @@ +import * as vscode from "vscode"; + +export function l10nMsg(message: string, args?: Record): string { + return vscode.l10n.t(message, args || {}); +} diff --git a/eslint.config.mjs b/eslint.config.mjs index 86d9dede..aa2d8617 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -2,12 +2,10 @@ import globals from "globals"; import typescriptEslint from "@typescript-eslint/eslint-plugin"; import tsParser from "@typescript-eslint/parser"; import path from "node:path"; -import { fileURLToPath } from "node:url"; import js from "@eslint/js"; import { FlatCompat } from "@eslint/eslintrc"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); +const __dirname = path.dirname(import.meta.dirname); const compat = new FlatCompat({ baseDirectory: __dirname, recommendedConfig: js.configs.recommended, diff --git a/locales/tinymist-rt.toml b/locales/tinymist-rt.toml new file mode 100644 index 00000000..760c987a --- /dev/null +++ b/locales/tinymist-rt.toml @@ -0,0 +1,22 @@ + +# The translation are partially generated by copilot + +[tinymist-query.code-action.exportHtml] +en = "Export HTML" +zh = "导出 HTML" + +[tinymist-query.code-action.exportPdf] +en = "Export PDF" +zh = "导出 PDF" + +[tinymist-query.code-action.more] +en = "More .." +zh = "更多 .." + +[tinymist-query.code-action.preview] +en = "Preview" +zh = "预览" + +[tinymist-query.code-action.profile] +en = "Profile" +zh = "性能分析" diff --git a/locales/tinymist-vscode-rt.toml b/locales/tinymist-vscode-rt.toml new file mode 100644 index 00000000..af7288e8 --- /dev/null +++ b/locales/tinymist-vscode-rt.toml @@ -0,0 +1,6 @@ + +# The translation are partially generated by copilot + +[" (Preview)"] +en = " (Preview)" +zh = "（预览）" diff --git a/package.json b/package.json index 30aef2d7..a6cfadad 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,10 @@ "scripts": { "build:editor-tools": "cd tools/editor-tools/ && yarn run build", "build:preview": "cd tools/typst-preview-frontend && yarn run build && rimraf ../../crates/tinymist-assets/src/typst-preview.html && cpr ./dist/index.html ../../crates/tinymist-assets/src/typst-preview.html", - "build:l10n": "node scripts/build-l10n.mjs", + "build:l10n": "yarn extract:l10n && node scripts/build-l10n.mjs", + "extract:l10n": "yarn extract:l10n:ts && yarn extract:l10n:rs", + "extract:l10n:ts": "cargo run --release --bin tinymist-l10n -- --kind ts --dir ./editors/vscode --output ./locales/tinymist-vscode-rt.toml", + "extract:l10n:rs": "cargo run --release --bin tinymist-l10n -- --kind rs --dir ./crates --output ./locales/tinymist-rt.toml && rimraf ./crates/tinymist-assets/src/tinymist-rt.toml && cpr ./locales/tinymist-rt.toml ./crates/tinymist-assets/src/tinymist-rt.toml", "maintainers": "typst query MAINTAINERS.typ \"\" --pretty --one --field value --input=action=help", "docs": "shiroa serve --font-path assets/fonts -w . docs/tinymist", "docs:pdf": "cargo run --bin tinymist --release -- compile --font-path assets/fonts --root . docs/tinymist/ebook.typ", diff --git a/scripts/build-l10n.mjs b/scripts/build-l10n.mjs index 3aefa937..74360437 100644 --- a/scripts/build-l10n.mjs +++ b/scripts/build-l10n.mjs @@ -8,21 +8,31 @@ import * as fs from "fs"; import * as path from "path"; -const __dirname = new URL(".", import.meta.url).toString().replace("file:///", ""); -const projectRoot = path.resolve(__dirname, ".."); +const projectRoot = path.resolve(import.meta.dirname, ".."); -function translate(input, output) { - const data = fs.readFileSync(path.resolve(projectRoot, input), "utf-8"); - const lines = data - .split("\n") - .map((line) => line.trim()) - .filter((line) => !line.startsWith("#") && line.length > 0); +/** + * + * @param {string} output + * @param {string[]} inputs + * @returns + */ +function translate(output, kind, inputs) { + const lines = inputs.flatMap((input) => + fs + .readFileSync(path.resolve(projectRoot, input), "utf-8") + .split("\n") + .map((line) => line.trim()) + .filter((line) => !line.startsWith("#") && line.length > 0), + ); const translations = {}; let key = ""; for (let line of lines) { if (line.startsWith("[")) { key = line.substring(1, line.length - 1); + if (key.startsWith('"')) { + key = JSON.parse(key); + } } else { const equalIndex = line.indexOf("="); const lang = line.substring(0, equalIndex).trim(); @@ -42,21 +52,34 @@ function translate(input, output) { const langRest = langs.filter((lang) => lang !== "en"); - const langEnPath = path.resolve(projectRoot, `${output}/package.nls.json`); + const langDir = path.resolve(projectRoot, output); + fs.mkdirSync(langDir, { recursive: true }); + + const langEnPath = `${langDir}/${kind}.json`; const langEnData = translations["en"]; fs.writeFileSync(langEnPath, JSON.stringify(langEnData, null, 2)); for (let lang of langRest) { - const langPath = path.resolve(projectRoot, `${output}/package.nls.${lang}.json`); + const langPath = `${langDir}/${kind}.${lang}.json`; const langData = translations[lang]; - fs.writeFileSync(langPath, JSON.stringify(langData, null, 2)); + const langPack = JSON.stringify(langData, null, 2); + + fs.writeFileSync(langPath, langPack); + + // alias zh-cn + if (kind === "bundle.l10n" && lang === "zh") { + const langZhCNPath = `${langDir}/${kind}.zh-cn.json`; + fs.writeFileSync(langZhCNPath, langPack); + } } return translations; } +// todo: verify using rust function genVscodeExt() { - const translations = translate("locales/tinymist-vscode.toml", "editors/vscode"); + const translations = translate("editors/vscode", "package.nls", ["locales/tinymist-vscode.toml"]); + translate("editors/vscode/l10n", "bundle.l10n", ["locales/tinymist-vscode-rt.toml"]); const pat = /\%(extension\.tinymist\..*?)\%/g; const data = fs.readFileSync(path.resolve(projectRoot, "editors/vscode/package.json"), "utf-8"); diff --git a/scripts/link-docs.mjs b/scripts/link-docs.mjs index dd1669e3..9a952f72 100644 --- a/scripts/link-docs.mjs +++ b/scripts/link-docs.mjs @@ -1,10 +1,8 @@ import { resolve, basename } from "path"; -import { fileURLToPath } from "url"; import * as fs from "fs"; import { execSync } from "child_process"; -const filename = fileURLToPath(import.meta.url); -const root = resolve(filename, "../.."); +const root = resolve(import.meta.dirname, "../.."); const dry = process.argv.includes("--dry"); const bytes2utf8 = new TextDecoder(); @@ -59,7 +57,7 @@ const convert = async (inp, out, opts) => { imageCnt += 1; fs.writeFileSync(resolve(assetsDir, fileName), base64Decode(content)); return `"./assets/images/${fileName}"`; - } + }, ); await fs.promises.writeFile(output, outputContent); @@ -70,13 +68,9 @@ const main = async () => { convert("docs/tinymist/introduction.typ", "README.md", { before: "# Tinymist\n\n", }), - convert( - "docs/tinymist/release-instruction.typ", - "docs/release-instruction.md", - { - before: "# Release Instructions\n\n", - } - ), + convert("docs/tinymist/release-instruction.typ", "docs/release-instruction.md", { + before: "# Release Instructions\n\n", + }), convert("docs/tinymist/frontend/emacs.typ", "editors/emacs/README.md", { before: "# Tinymist Emacs Support for Typst\n\n", }), @@ -86,11 +80,9 @@ const main = async () => { convert("docs/tinymist/frontend/neovim.typ", "editors/neovim/README.md", { before: "# Tinymist Neovim Support for Typst\n\n", }), - convert( - "docs/tinymist/frontend/sublime-text.typ", - "editors/sublime-text/README.md", - { before: "# Tinymist Sublime Support for Typst\n\n" } - ), + convert("docs/tinymist/frontend/sublime-text.typ", "editors/sublime-text/README.md", { + before: "# Tinymist Sublime Support for Typst\n\n", + }), convert("docs/tinymist/frontend/vscode.typ", "editors/vscode/README.md", { before: "# Tinymist Typst VS Code Extension\n\n", }), diff --git a/syntaxes/textmate/main.mts b/syntaxes/textmate/main.mts index c0ed36c7..c17d0213 100644 --- a/syntaxes/textmate/main.mts +++ b/syntaxes/textmate/main.mts @@ -14,7 +14,6 @@ import { blockRaw, blockRawGeneral, blockRawLangs, inlineRaw } from "./fenced.mj import * as fs from "fs"; import * as path from "path"; -import { fileURLToPath } from "node:url"; import { FIXED_LENGTH_LOOK_BEHIND, POLYFILL_P_XID, @@ -1465,8 +1464,6 @@ export const typst: textmate.Grammar = { }; function generate() { - const dirname = fileURLToPath(new URL(".", import.meta.url)); - let compiled = textmate.compile(typst); if (POLYFILL_P_XID) { @@ -1492,7 +1489,7 @@ function generate() { // dump to file fs.writeFileSync( - path.join(dirname, "../typst.tmLanguage.json"), + path.join(import.meta.dirname, "../typst.tmLanguage.json"), JSON.stringify({ $schema: "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", scopeName: "source.typst", @@ -1504,7 +1501,7 @@ function generate() { // dump to file fs.writeFileSync( - path.join(dirname, "../typst-code.tmLanguage.json"), + path.join(import.meta.dirname, "../typst-code.tmLanguage.json"), JSON.stringify({ $schema: "https://raw.githubusercontent.com/martinring/tmlanguage/master/tmlanguage.json", scopeName: "source.typst-code",