diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 841c80db..8871591d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -332,28 +332,28 @@ jobs: publish: runs-on: ubuntu-latest needs: [build] - if: success() && startsWith(github.ref, 'refs/tags/') + if: success() && startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') steps: - uses: actions/download-artifact@v4 - name: Install deps run: yarn install - name: Deploy to VS Code Marketplace - if: "!contains(github.ref, 'rc')" + if: "(endsWith(github.ref, '0') || endsWith(github.ref, '2') || endsWith(github.ref, '4') || endsWith(github.ref, '6') || endsWith(github.ref, '8'))" run: npx @vscode/vsce publish --packagePath $(find . -type f -iname 'tinymist-*.vsix') --skip-duplicate env: VSCE_PAT: ${{ secrets.VSCODE_MARKETPLACE_TOKEN }} - name: Deploy to OpenVSX - if: "!contains(github.ref, 'rc')" + if: "(endsWith(github.ref, '0') || endsWith(github.ref, '2') || endsWith(github.ref, '4') || endsWith(github.ref, '6') || endsWith(github.ref, '8'))" run: npx ovsx publish --packagePath $(find . -type f -iname 'tinymist-*.vsix') --skip-duplicate env: OVSX_PAT: ${{ secrets.OPENVSX_ACCESS_TOKEN }} - name: Deploy to VS Code Marketplace (Pre Release) - if: contains(github.ref, 'rc') + if: "(endsWith(github.ref, '1') || endsWith(github.ref, '3') || endsWith(github.ref, '5') || endsWith(github.ref, '7') || endsWith(github.ref, '9'))" run: npx @vscode/vsce publish --packagePath $(find . -type f -iname 'tinymist-*.vsix') --skip-duplicate --pre-release env: VSCE_PAT: ${{ secrets.VSCODE_MARKETPLACE_TOKEN }} - name: Deploy to OpenVSX (Pre Release) - if: contains(github.ref, 'rc') + if: "(endsWith(github.ref, '1') || endsWith(github.ref, '3') || endsWith(github.ref, '5') || endsWith(github.ref, '7') || endsWith(github.ref, '9'))" run: npx ovsx publish --packagePath $(find . -type f -iname 'tinymist-*.vsix') --skip-duplicate --pre-release env: OVSX_PAT: ${{ secrets.OPENVSX_ACCESS_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 589a832a..48f3ee31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -492,9 +492,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -502,9 +502,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -517,9 +517,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.5.28" +version = "4.5.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b378c786d3bde9442d2c6dd7e6080b2a818db2b96e30d6e7f1b6d224eb617d3" +checksum = "8937760c3f4c60871870b8c3ee5f9b30771f792a7045c48bcbba999d7d6b3b8e" dependencies = [ "clap", ] @@ -546,9 +546,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2608,9 +2608,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" [[package]] name = "postcard" @@ -3306,9 +3306,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -3679,7 +3679,7 @@ dependencies = [ [[package]] name = "sync-lsp" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "anyhow", "clap", @@ -3819,7 +3819,7 @@ dependencies = [ [[package]] name = "tests" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "insta", "lsp-server", @@ -3916,7 +3916,7 @@ dependencies = [ [[package]] name = "tinymist" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "anyhow", "async-trait", @@ -3956,7 +3956,7 @@ dependencies = [ "serde_json", "serde_yaml", "sync-lsp", - "tinymist-assets 0.11.22-rc1 (registry+https://github.com/rust-lang/crates.io-index)", + "tinymist-assets 0.11.22 (registry+https://github.com/rust-lang/crates.io-index)", "tinymist-query", "tinymist-render", "tinymist-world", @@ -3982,7 +3982,7 @@ dependencies = [ [[package]] name = "tinymist-analysis" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "base64 0.22.1", "comemo 0.4.0", @@ -3999,17 +3999,17 @@ dependencies = [ [[package]] name = "tinymist-assets" -version = "0.11.22-rc1" +version = "0.11.22" [[package]] name = "tinymist-assets" -version = "0.11.22-rc1" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0b973646edfc3ee0d27b7344de27de9c60c3e765e46573918b9f6d7d577b55" +checksum = "133e31d88d00f03791f99009c4102a14b2b1a7474e8179ca7e2e42806c7134ef" [[package]] name = "tinymist-query" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "anyhow", "base64 0.22.1", @@ -4060,7 +4060,7 @@ dependencies = [ [[package]] name = "tinymist-render" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "base64 0.22.1", "log", @@ -4071,7 +4071,7 @@ dependencies = [ [[package]] name = "tinymist-world" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "anyhow", "chrono", @@ -4086,7 +4086,7 @@ dependencies = [ "serde", "serde_json", "tar", - "tinymist-assets 0.11.22-rc1 (registry+https://github.com/rust-lang/crates.io-index)", + "tinymist-assets 0.11.22 (registry+https://github.com/rust-lang/crates.io-index)", "typst-assets", ] @@ -4348,7 +4348,7 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typlite" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "base64 0.22.1", "comemo 0.4.0", @@ -4492,7 +4492,7 @@ dependencies = [ [[package]] name = "typst-preview" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "clap", "comemo 0.4.0", @@ -4505,7 +4505,7 @@ dependencies = [ "reflexo-vec2svg", "serde", "serde_json", - "tinymist-assets 0.11.22-rc1 (registry+https://github.com/rust-lang/crates.io-index)", + "tinymist-assets 0.11.22 (registry+https://github.com/rust-lang/crates.io-index)", "tokio", "typst", "typst-assets", @@ -4533,7 +4533,7 @@ dependencies = [ [[package]] name = "typst-shim" -version = "0.11.22-rc1" +version = "0.11.22" dependencies = [ "cfg-if", "typst", @@ -4617,9 +4617,9 @@ dependencies = [ [[package]] name = "typstyle" -version = "0.11.32" +version = "0.11.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e79e77aac4d80934811be14abd1e2b0e7d3b4059653e24b841ab1ac5bae0" +checksum = "4c430c527e8048f19522448613e95da687ed64fb0a3b78969f0643e6e99a95ef" dependencies = [ "anyhow", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index bb57e15d..263b3b97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace.package] description = "An integrated language service for Typst." authors = ["Myriad-Dreamin ", "Nathan Varner"] -version = "0.11.22-rc1" +version = "0.11.22" edition = "2021" readme = "README.md" license = "Apache-2.0" @@ -137,7 +137,7 @@ insta = { version = "1.39", features = ["glob"] } # Our Own Crates typst-preview = { path = "./crates/typst-preview/" } -tinymist-assets = { version = "0.11.22-rc1" } +tinymist-assets = { version = "0.11.22" } tinymist = { path = "./crates/tinymist/" } tinymist-analysis = { path = "./crates/tinymist-analysis/" } tinymist-query = { path = "./crates/tinymist-query/" } diff --git a/crates/tinymist-query/src/docs/mod.rs b/crates/tinymist-query/src/docs/mod.rs index 775efdc2..b662bfb1 100644 --- a/crates/tinymist-query/src/docs/mod.rs +++ b/crates/tinymist-query/src/docs/mod.rs @@ -12,14 +12,16 @@ use std::sync::Arc; use comemo::Track; use ecow::{eco_vec, EcoString, EcoVec}; use indexmap::IndexSet; +use itertools::Itertools; use parking_lot::Mutex; +use reflexo::path::unix_slash; use serde::{Deserialize, Serialize}; use tinymist_world::base::{EntryState, ShadowApi, TaskInputs}; use tinymist_world::LspWorld; use typst::diag::{eco_format, StrResult}; use typst::engine::Route; use typst::eval::Tracer; -use typst::foundations::{Bytes, Value}; +use typst::foundations::{Bytes, Module, Value}; use typst::syntax::package::{PackageManifest, PackageSpec}; use typst::syntax::{FileId, Span, VirtualPath}; use typst::World; @@ -85,6 +87,8 @@ impl Docs { } } +type TypeRepr = Option<(/* short */ String, /* long */ String)>; + /// Describes a primary function signature. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DocSignature { @@ -95,7 +99,7 @@ pub struct DocSignature { /// The rest parameter. pub rest: Option, /// The return type. - pub ret_ty: Option<(String, String)>, + pub ret_ty: TypeRepr, } /// Describes a function parameter. @@ -106,7 +110,7 @@ pub struct DocParamSpec { /// Documentation for the parameter. pub docs: String, /// Inferred type of the parameter. - pub cano_type: Option<(String, String)>, + pub cano_type: TypeRepr, /// The parameter's default name as type. pub type_repr: Option, /// The parameter's default name as value. @@ -125,7 +129,7 @@ pub struct DocParamSpec { } /// Information about a symbol. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct SymbolInfoHead { /// The name of the symbol. pub name: EcoString, @@ -133,6 +137,12 @@ pub struct SymbolInfoHead { pub kind: EcoString, /// The location (file, start, end) of the symbol. pub loc: Option<(usize, usize, usize)>, + /// Is the symbol reexport + pub export_again: bool, + /// Is the symbol reexport + pub external_link: Option, + /// The one-line documentation of the symbol. + pub oneliner: Option, /// The raw documentation of the symbol. pub docs: Option, /// The signature of the symbol. @@ -142,6 +152,9 @@ pub struct SymbolInfoHead { /// The value of the symbol. #[serde(skip)] pub constant: Option, + /// The file owning the symbol. + #[serde(skip)] + pub fid: Option, /// The span of the symbol. #[serde(skip)] pub span: Option, @@ -163,6 +176,16 @@ pub struct SymbolInfo { pub children: EcoVec, } +/// Information about the symbols in a package. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SymbolsInfo { + /// The root module information. + #[serde(flatten)] + pub root: SymbolInfo, + /// The module accessible paths. + pub module_uses: HashMap>, +} + /// Information about a package. #[derive(Debug, Serialize, Deserialize)] pub struct PackageMeta { @@ -190,6 +213,18 @@ pub struct FileMeta { path: PathBuf, } +/// Parses the manifest of the package located at `package_path`. +pub fn get_manifest_id(spec: &PackageInfo) -> StrResult { + Ok(FileId::new( + Some(PackageSpec { + namespace: spec.namespace.clone(), + name: spec.name.clone(), + version: spec.version.parse()?, + }), + VirtualPath::new("typst.toml"), + )) +} + /// Parses the manifest of the package located at `package_path`. pub fn get_manifest(world: &LspWorld, toml_id: FileId) -> StrResult { let toml_data = world @@ -203,33 +238,160 @@ pub fn get_manifest(world: &LspWorld, toml_id: FileId) -> StrResult { + world: &'a LspWorld, + for_spec: Option<&'a PackageSpec>, + aliases: &'a mut HashMap>, + extras: &'a mut Vec, + route: Route<'a>, + root: FileId, + tracer: Tracer, +} + +impl ScanSymbolCtx<'_> { + fn module(&mut self, fid: FileId) -> StrResult { + let source = self.world.source(fid).map_err(|e| eco_format!("{e}"))?; + let route = self.route.track(); + let tracer = self.tracer.track_mut(); + let w: &dyn typst::World = self.world; + + typst::eval::eval(w.track(), route, tracer, &source).map_err(|e| eco_format!("{e:?}")) + } + + fn module_sym(&mut self, path: EcoVec<&str>, module: Module) -> SymbolInfo { + let key = module.name().to_owned(); + let site = Some(self.root); + let p = path.clone(); + self.sym(&key, p, site.as_ref(), &Value::Module(module)) + } + + fn sym( + &mut self, + key: &str, + path: EcoVec<&str>, + site: Option<&FileId>, + val: &Value, + ) -> SymbolInfo { + let mut head = create_head(self.world, key, val); + + if !matches!(&val, Value::Module(..)) { + if let Some((span, mod_fid)) = head.span.and_then(Span::id).zip(site) { + if span != *mod_fid { + head.export_again = true; + head.oneliner = head.docs.as_deref().map(oneliner).map(|e| e.to_owned()); + head.docs = None; + } + } + } + + let children = match val { + Value::Module(module) => module.file_id().and_then(|fid| { + // only generate docs for the same package + if fid.package() != self.for_spec { + return None; + } + + // !aliases.insert(fid) + let aliases_vec = self.aliases.entry(fid).or_default(); + let is_fresh = aliases_vec.is_empty(); + aliases_vec.push(path.iter().join(".")); + + if !is_fresh { + log::debug!("found module: {path:?} (reexport)"); + return None; + } + + log::debug!("found module: {path:?}"); + + let symbols = module.scope().iter(); + let symbols = symbols + .map(|(k, v)| { + let mut path = path.clone(); + path.push(k); + self.sym(k, path.clone(), Some(&fid), v) + }) + .collect(); + Some(symbols) + }), + _ => None, + }; + + // Insert module that is not exported + if let Some(fid) = head.fid { + // only generate docs for the same package + if fid.package() == self.for_spec { + let av = self.aliases.entry(fid).or_default(); + if av.is_empty() { + let m = self.module(fid); + let mut path = path.clone(); + path.push("-"); + path.push(key); + + log::debug!("found internal module: {path:?}"); + if let Ok(m) = m { + let msym = self.module_sym(path, m); + self.extras.push(msym) + } + } + } + } + + let children = children.unwrap_or_default(); + SymbolInfo { head, children } + } +} + /// List all symbols in a package. -pub fn list_symbols(world: &LspWorld, spec: &PackageInfo) -> StrResult { - let toml_id = FileId::new( - Some(PackageSpec { - namespace: spec.namespace.clone(), - name: spec.name.clone(), - version: spec.version.parse()?, - }), - VirtualPath::new("typst.toml"), - ); +pub fn list_symbols(world: &LspWorld, spec: &PackageInfo) -> StrResult { + let toml_id = get_manifest_id(spec)?; let manifest = get_manifest(world, toml_id)?; - let entry_point = toml_id.join(&manifest.package.entrypoint); - let source = world.source(entry_point).map_err(|e| eco_format!("{e}"))?; - let route = Route::default(); - let mut tracer = Tracer::default(); - let w: &dyn typst::World = world; - - let src = typst::eval::eval(w.track(), route.track(), tracer.track_mut(), &source) - .map_err(|e| eco_format!("{e:?}"))?; - let for_spec = PackageSpec { namespace: spec.namespace.clone(), name: spec.name.clone(), version: spec.version.parse()?, }; - Ok(symbol(world, Some(&for_spec), "root", &Value::Module(src))) + let mut aliases = HashMap::new(); + let mut extras = vec![]; + let entry_point = toml_id.join(&manifest.package.entrypoint); + + let mut scan_ctx = ScanSymbolCtx { + world, + root: entry_point, + for_spec: Some(&for_spec), + aliases: &mut aliases, + extras: &mut extras, + route: Route::default(), + tracer: Tracer::default(), + }; + + let src = scan_ctx.module(entry_point)?; + let mut symbols = scan_ctx.module_sym(eco_vec![], src); + + let module_uses = aliases + .into_iter() + .map(|(k, mut v)| { + v.sort_by(|a, b| a.len().cmp(&b.len()).then(a.cmp(b))); + (file_id_repr(k), v.into()) + }) + .collect(); + + log::debug!("module_uses: {module_uses:#?}",); + + symbols.children.extend(extras); + + Ok(SymbolsInfo { + root: symbols, + module_uses, + }) +} + +fn file_id_repr(k: FileId) -> String { + if let Some(p) = k.package() { + format!("{p}{}", unix_slash(k.vpath().as_rooted_path())) + } else { + unix_slash(k.vpath().as_rooted_path()) + } } fn jbase64(s: &T) -> String { @@ -269,6 +431,135 @@ fn convert_docs(world: &LspWorld, content: &str) -> StrResult { Ok(conv) } +fn identify_docs(kind: &str, content: &str) -> StrResult { + match kind { + "function" => identify_tidy_func_docs(content).map(Docs::Function), + "variable" => identify_tidy_var_docs(content).map(Docs::Variable), + "module" => identify_tidy_module_docs(content).map(Docs::Module), + _ => Err(eco_format!("unknown kind {kind}")), + } +} + +type TypeInfo = (Arc, Arc); + +fn docs_signature( + ctx: &mut AnalysisContext, + type_info: Option<&TypeInfo>, + sym: &SymbolInfo, + e: Value, + doc_ty: &mut impl FnMut(Option<&Ty>) -> TypeRepr, +) -> Option { + let func = match &e { + Value::Func(f) => f, + _ => return None, + }; + + // todo: documenting with bindings + use typst::foundations::func::Repr; + let mut func = func; + loop { + match func.inner() { + Repr::Element(..) | Repr::Native(..) => { + break; + } + Repr::With(w) => { + func = &w.0; + } + Repr::Closure(..) => { + break; + } + } + } + + let sig = analyze_dyn_signature(ctx, func.clone()); + let type_sig = type_info.and_then(|(def_use, ty_chk)| { + let def_fid = func.span().id()?; + let def_ident = IdentRef { + name: sym.head.name.clone(), + range: sym.head.name_range.clone()?, + }; + let (def_id, _) = def_use.get_def(def_fid, &def_ident)?; + ty_chk.type_of_def(def_id) + }); + let type_sig = type_sig.and_then(|type_sig| type_sig.sig_repr(true)); + + let pos_in = sig + .primary() + .pos + .iter() + .enumerate() + .map(|(i, pos)| (pos, type_sig.as_ref().and_then(|sig| sig.pos(i)))); + let named_in = sig + .primary() + .named + .iter() + .map(|x| (x, type_sig.as_ref().and_then(|sig| sig.named(x.0)))); + let rest_in = sig + .primary() + .rest + .as_ref() + .map(|x| (x, type_sig.as_ref().and_then(|sig| sig.rest_param()))); + + let ret_in = type_sig + .as_ref() + .and_then(|sig| sig.body.as_ref()) + .or_else(|| sig.primary().ret_ty.as_ref()); + + let pos = pos_in + .map(|(param, ty)| DocParamSpec { + name: param.name.as_ref().to_owned(), + docs: param.docs.as_ref().to_owned(), + cano_type: doc_ty(ty), + type_repr: param.type_repr.clone(), + expr: param.expr.clone(), + positional: param.positional, + named: param.named, + variadic: param.variadic, + settable: param.settable, + }) + .collect(); + + let named = named_in + .map(|((name, param), ty)| { + ( + name.as_ref().to_owned(), + DocParamSpec { + name: param.name.as_ref().to_owned(), + docs: param.docs.as_ref().to_owned(), + cano_type: doc_ty(ty), + type_repr: param.type_repr.clone(), + expr: param.expr.clone(), + positional: param.positional, + named: param.named, + variadic: param.variadic, + settable: param.settable, + }, + ) + }) + .collect(); + + let rest = rest_in.map(|(param, ty)| DocParamSpec { + name: param.name.as_ref().to_owned(), + docs: param.docs.as_ref().to_owned(), + cano_type: doc_ty(ty), + type_repr: param.type_repr.clone(), + expr: param.expr.clone(), + positional: param.positional, + named: param.named, + variadic: param.variadic, + settable: param.settable, + }); + + let ret_ty = doc_ty(ret_in); + + Some(DocSignature { + pos, + named, + rest, + ret_ty, + }) +} + #[derive(Serialize, Deserialize)] struct ConvertResult { errors: Vec, @@ -281,17 +572,18 @@ pub fn generate_md_docs( spec: &PackageInfo, ) -> StrResult { log::info!("generate_md_docs {spec:?}"); - let toml_id = FileId::new( - Some(PackageSpec { - namespace: spec.namespace.clone(), - name: spec.name.clone(), - version: spec.version.parse()?, - }), - VirtualPath::new("typst.toml"), - ); + let toml_id = get_manifest_id(spec)?; + + let for_spec = PackageSpec { + namespace: spec.namespace.clone(), + name: spec.name.clone(), + version: spec.version.parse()?, + }; let mut md = String::new(); - let sym = list_symbols(world, spec)?; + let SymbolsInfo { root, module_uses } = list_symbols(world, spec)?; + + log::debug!("module_uses: {module_uses:#?}"); let title = format!("@{}/{}:{}", spec.namespace, spec.name, spec.version); @@ -314,27 +606,62 @@ pub fn generate_md_docs( let package_meta = jbase64(&meta); let _ = writeln!(md, ""); - let mut key = 0; - - let mut modules_to_generate = vec![(EcoString::new(), sym.head.name.clone(), sym)]; + let mut modules_to_generate = vec![(root.head.name.clone(), root)]; let mut generated_modules = HashSet::new(); - let mut file_ids = IndexSet::new(); + let mut file_ids: IndexSet = IndexSet::new(); + + // let aka = module_uses[&file_id_repr(fid.unwrap())].clone(); + // let primary = &aka[0]; + let mut primary_aka_cache = HashMap::>::new(); + let mut akas = |fid: FileId| { + primary_aka_cache + .entry(fid) + .or_insert_with(|| { + module_uses + .get(&file_id_repr(fid)) + .unwrap_or_else(|| panic!("no module uses for {}", file_id_repr(fid))) + .clone() + }) + .clone() + }; + + // todo: extend this cache idea for all crate? + #[allow(clippy::mutable_key_type)] + let mut describe_cache = HashMap::::new(); + let mut doc_ty = |ty: Option<&Ty>| { + let ty = ty?; + let short = { + describe_cache + .entry(ty.clone()) + .or_insert_with(|| ty.describe().unwrap_or_else(|| "unknown".to_string())) + .clone() + }; + + Some((short, format!("{ty:?}"))) + }; while !modules_to_generate.is_empty() { - for (prefix, parent_ident, sym) in std::mem::take(&mut modules_to_generate) { + for (parent_ident, sym) in std::mem::take(&mut modules_to_generate) { // parent_ident, symbols let symbols = sym.children; - if !prefix.is_empty() { - let _ = writeln!(md, "---\n## Module: {prefix}"); - } let module_val = sym.head.value.as_ref().unwrap(); let module = match module_val { Value::Module(m) => m, _ => todo!(), }; - let fid = module.file_id(); + let aka = fid.map(&mut akas).unwrap_or_default(); + + // It is (primary) known to safe as a part of HTML string, so we don't have to + // do sanitization here. + let primary = aka.first().cloned().unwrap_or_default(); + if !primary.is_empty() { + let _ = writeln!(md, "---\n## Module: {primary}"); + } + + log::debug!("module: {primary} -- {parent_ident}"); + let type_info = None.or_else(|| { let file_id = fid?; let src = world.source(file_id).ok()?; @@ -352,14 +679,16 @@ pub fn generate_md_docs( name: EcoString, loc: Option, parent_ident: EcoString, + aka: EcoVec, } let m = jbase64(&ModuleInfo { - prefix: prefix.clone(), + prefix: primary.as_str().into(), name: sym.head.name.clone(), loc: persist_fid, parent_ident: parent_ident.clone(), + aka, }); - let _ = writeln!(md, ""); + let _ = writeln!(md, ""); for mut sym in symbols { let span = sym.head.span.and_then(|v| { @@ -370,27 +699,25 @@ pub fn generate_md_docs( Some((fid, rng.start, rng.end)) }) }); + let sym_fid = sym.head.fid; + let sym_fid = sym_fid.or_else(|| sym.head.span.and_then(Span::id)).or(fid); + let span = span.or_else(|| { + let fid = sym_fid?; + Some((file_ids.insert_full(fid).0, 0, 0)) + }); sym.head.loc = span; + let sym_value = sym.head.value.clone(); + let signature = + sym_value.and_then(|e| docs_signature(ctx, type_info, &sym, e, &mut doc_ty)); + sym.head.signature = signature; + let mut convert_err = None; if let Some(docs) = &sym.head.docs { match convert_docs(world, docs) { Ok(content) => { - let docs = match sym.head.kind.as_str() { - "function" => { - let t = identify_tidy_func_docs(&content).ok(); - t.map(Docs::Function).unwrap_or(Docs::Plain(content)) - } - "variable" => { - let t = identify_tidy_var_docs(&content).ok(); - t.map(Docs::Variable).unwrap_or(Docs::Plain(content)) - } - "module" => { - let t = identify_tidy_module_docs(&content).ok(); - t.map(Docs::Module).unwrap_or(Docs::Plain(content)) - } - _ => Docs::Plain(content), - }; + let docs = identify_docs(sym.head.kind.as_str(), &content) + .unwrap_or(Docs::Plain(content)); sym.head.parsed_docs = Some(docs.clone()); sym.head.docs = None; @@ -405,115 +732,41 @@ pub fn generate_md_docs( } } - let signature = - match &sym.head.parsed_docs { - Some(Docs::Function(TidyFuncDocs { - params, return_ty, .. - })) => sym.head.value.clone().and_then(|e| { - let func = match e { - Value::Func(f) => f, - _ => return None, - }; - let sig = analyze_dyn_signature(ctx, func.clone()); - let type_sig = type_info.and_then(|(def_use, ty_chk)| { - let def_fid = func.span().id()?; - let def_ident = IdentRef { - name: sym.head.name.clone(), - range: sym.head.name_range.clone()?, - }; - let (def_id, _) = def_use.get_def(def_fid, &def_ident)?; - ty_chk.type_of_def(def_id) - }); - let type_sig = type_sig.and_then(|type_sig| type_sig.sig_repr(true)); + let ident = if !primary.is_empty() { + eco_format!("symbol-{}-{primary}.{}", sym.head.kind, sym.head.name) + } else { + eco_format!("symbol-{}-{}", sym.head.kind, sym.head.name) + }; + let _ = writeln!(md, "### {}: {} in {primary}", sym.head.kind, sym.head.name); - let pos_in = sig.primary().pos.iter().enumerate().map(|(i, pos)| { - (pos, type_sig.as_ref().and_then(|sig| sig.pos(i))) - }); - let named_in = sig - .primary() - .named - .iter() - .map(|x| (x, type_sig.as_ref().and_then(|sig| sig.named(x.0)))); - let rest_in = - sig.primary().rest.as_ref().map(|x| { - (x, type_sig.as_ref().and_then(|sig| sig.rest_param())) - }); + if sym.head.export_again { + let sub_fid = sym.head.fid; + if let Some(fid) = sub_fid { + let lnk = if fid.package() == Some(&for_spec) { + let sub_aka = akas(fid); + let sub_primary = sub_aka.first().cloned().unwrap_or_default(); + sym.head.external_link = Some(format!( + "#symbol-{}-{sub_primary}.{}", + sym.head.kind, sym.head.name + )); + format!("#{}-{}-in-{sub_primary}", sym.head.kind, sym.head.name) + .replace(".", "") + } else if let Some(spec) = fid.package() { + let lnk = format!( + "https://typst.app/universe/package/{}/{}", + spec.name, spec.version + ); + sym.head.external_link = Some(lnk.clone()); + lnk + } else { + let lnk: String = "https://typst.app/docs".into(); + sym.head.external_link = Some(lnk.clone()); + lnk + }; + let _ = writeln!(md, "[Symbol Docs]({lnk})\n"); + } + } - let ret_in = type_sig - .as_ref() - .and_then(|sig| sig.body.as_ref()) - .or_else(|| sig.primary().ret_ty.as_ref()); - - let doc_ty = |ty: Option<&Ty>| { - ty.and_then(|ty| ty.describe().map(|e| (e, format!("{ty:?}")))) - }; - - let _ = params; - let _ = return_ty; - - let pos = pos_in - .map(|(param, ty)| DocParamSpec { - name: param.name.as_ref().to_owned(), - docs: param.docs.as_ref().to_owned(), - cano_type: doc_ty(ty), - type_repr: param.type_repr.clone(), - expr: param.expr.clone(), - positional: param.positional, - named: param.named, - variadic: param.variadic, - settable: param.settable, - }) - .collect(); - - let named = named_in - .map(|((name, param), ty)| { - ( - name.as_ref().to_owned(), - DocParamSpec { - name: param.name.as_ref().to_owned(), - docs: param.docs.as_ref().to_owned(), - cano_type: doc_ty(ty), - type_repr: param.type_repr.clone(), - expr: param.expr.clone(), - positional: param.positional, - named: param.named, - variadic: param.variadic, - settable: param.settable, - }, - ) - }) - .collect(); - - let rest = rest_in.map(|(param, ty)| DocParamSpec { - name: param.name.as_ref().to_owned(), - docs: param.docs.as_ref().to_owned(), - cano_type: doc_ty(ty), - type_repr: param.type_repr.clone(), - expr: param.expr.clone(), - positional: param.positional, - named: param.named, - variadic: param.variadic, - settable: param.settable, - }); - - let ret_ty = doc_ty(ret_in); - - Some(DocSignature { - pos, - named, - rest, - ret_ty, - }) - }), - _ => None, - }; - - sym.head.signature = signature; - - let _ = writeln!(md, "### {}: {}", sym.head.kind, sym.head.name); - - let ident = eco_format!("symbol-{}-{}-{key}", sym.head.kind, sym.head.name); - key += 1; let head = jbase64(&sym.head); let _ = writeln!(md, ""); @@ -555,7 +808,10 @@ pub fn generate_md_docs( (None, None) => {} } - if let Some(docs) = &sym.head.docs { + let plain_docs = sym.head.docs.as_deref(); + let plain_docs = plain_docs.or(sym.head.oneliner.as_deref()); + + if let Some(docs) = plain_docs { let contains_code = docs.contains("```"); if contains_code { let _ = writeln!(md, "`````typ"); @@ -567,23 +823,29 @@ pub fn generate_md_docs( } if !sym.children.is_empty() { - let mut full_path = prefix.clone(); - if !full_path.is_empty() { - full_path.push_str("."); - } - full_path.push_str(&sym.head.name); - let link = format!("Module-{full_path}").replace(".", "-"); - let _ = writeln!(md, "[Module Docs](#{link})\n"); + let sub_fid = sym.head.fid; + log::debug!("sub_fid: {sub_fid:?}"); + match sub_fid { + Some(fid) => { + let aka = akas(fid); + let primary = aka.first().cloned().unwrap_or_default(); + let link = format!("module-{primary}").replace(".", ""); + let _ = writeln!(md, "[Module Docs](#{link})\n"); - if generated_modules.insert(full_path.clone()) { - modules_to_generate.push((full_path, ident.clone(), sym)); + if generated_modules.insert(fid) { + modules_to_generate.push((ident.clone(), sym)); + } + } + None => { + let _ = writeln!(md, "A Builtin Module"); + } } } let _ = writeln!(md, ""); } - let _ = writeln!(md, ""); + let _ = writeln!(md, ""); } } @@ -638,30 +900,9 @@ fn kind_of(val: &Value) -> EcoString { .into() } -fn symbol(world: &LspWorld, for_spec: Option<&PackageSpec>, key: &str, val: &Value) -> SymbolInfo { - let children = match val { - Value::Module(module) => { - // only generate docs for the same package - if module.file_id().map_or(true, |e| e.package() != for_spec) { - eco_vec![] - } else { - let symbols = module.scope().iter(); - symbols - .map(|(k, v)| symbol(world, for_spec, k, v)) - .collect() - } - } - _ => eco_vec![], - }; - SymbolInfo { - head: create_head(world, key, val), - children, - } -} - fn create_head(world: &LspWorld, k: &str, v: &Value) -> SymbolInfoHead { let kind = kind_of(v); - let (docs, name_range, span) = match v { + let (docs, name_range, fid, span) = match v { Value::Func(f) => { let mut span = None; let mut name_range = None; @@ -677,29 +918,36 @@ fn create_head(world: &LspWorld, k: &str, v: &Value) -> SymbolInfoHead { find_docs_of(&source, def) }); - (docs, name_range, span.or(Some(f.span()))) + let s = span.or(Some(f.span())); + + (docs, name_range, s.and_then(Span::id), s) } - _ => (None, None, None), + Value::Module(m) => (None, None, m.file_id(), None), + _ => Default::default(), }; SymbolInfoHead { name: k.to_string().into(), kind, - loc: None, constant: None.or_else(|| match v { Value::Func(_) => None, t => Some(truncated_doc_repr(t)), }), - signature: None, - parsed_docs: None, docs, name_range, + fid, span, value: Some(v.clone()), + ..Default::default() } } -// todo: hover with `with_stack` +/// Extract the first line of documentation. +fn oneliner(docs: &str) -> &str { + docs.lines().next().unwrap_or_default() +} + +// todo: hover with `with_stack`, todo: merge with hover tooltip struct ParamTooltip<'a>(&'a DocSignature); impl<'a> fmt::Display for ParamTooltip<'a> { diff --git a/crates/tinymist/src/cmd.rs b/crates/tinymist/src/cmd.rs index 6baa69a8..537e2ccb 100644 --- a/crates/tinymist/src/cmd.rs +++ b/crates/tinymist/src/cmd.rs @@ -3,6 +3,7 @@ use std::ops::Deref; use std::path::PathBuf; +use base::TaskInputs; use lsp_server::RequestId; use lsp_types::*; use reflexo_typst::error::prelude::*; @@ -12,7 +13,7 @@ use task::TraceParams; use tinymist_assets::TYPST_PREVIEW_HTML; use tinymist_query::docs::PackageInfo; use tinymist_query::{ExportKind, PageSelection}; -use typst::diag::{EcoString, StrResult}; +use typst::diag::{eco_format, EcoString, StrResult}; use typst::syntax::package::{PackageSpec, VersionlessPackageSpec}; use super::server::*; @@ -578,6 +579,25 @@ impl LanguageState { let snap = snap.receive().await.map_err(z_internal_error)?; let w = snap.world.as_ref(); + let entry: StrResult = Ok(()).and_then(|_| { + let toml_id = tinymist_query::docs::get_manifest_id(&info)?; + let toml_path = w.path_for_id(toml_id)?; + let pkg_root = toml_path.parent().ok_or_else(|| { + eco_format!("cannot get package root (parent of {toml_path:?})") + })?; + + let manifest = tinymist_query::docs::get_manifest(w, toml_id)?; + let entry_point = toml_id.join(&manifest.package.entrypoint); + + Ok(EntryState::new_rooted(pkg_root.into(), Some(entry_point))) + }); + let entry = entry.map_err(|e| internal_error(e.to_string()))?; + + let w = &snap.world.task(TaskInputs { + entry: Some(entry), + inputs: None, + }); + let res = handle.run_analysis(w, |a| { let symbols = tinymist_query::docs::generate_md_docs(a, w, &info) .map_err(map_string_err("failed to list symbols")) diff --git a/editors/vscode/package.json b/editors/vscode/package.json index 1e60d8db..8dd0ce79 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -1,6 +1,6 @@ { "name": "tinymist", - "version": "0.11.20", + "version": "0.11.22", "description": "An integrated language service for Typst", "categories": [ "Programming Languages", diff --git a/syntaxes/textmate/package.json b/syntaxes/textmate/package.json index d74e612b..f84d30ac 100644 --- a/syntaxes/textmate/package.json +++ b/syntaxes/textmate/package.json @@ -1,6 +1,6 @@ { "name": "typst-textmate", - "version": "0.11.20", + "version": "0.11.22", "private": true, "scripts": { "compile": "npx tsc && node ./dist/main.mjs", diff --git a/tools/editor-tools/src/features/docs.ts b/tools/editor-tools/src/features/docs.ts index 099e5615..9aade0f2 100644 --- a/tools/editor-tools/src/features/docs.ts +++ b/tools/editor-tools/src/features/docs.ts @@ -125,6 +125,8 @@ async function recoverDocsStructure(content: string) { break; case TokenKind.PackageEnd: const pkg = current; + pkg.data = pkg.data || {}; + pkg.data["pkgEndData"] = token[1]; current = structStack.pop()!; currentPkg = packageStack.pop()!; current.children.push(pkg); @@ -274,19 +276,82 @@ async function base64ToUtf8(base64: string) { return await res.text(); } -function getKnownModules(v: DocElement, s: Set) { - for (const child of v.children) { - if (child.kind === DocKind.Module) { - s.add(child.id); - } - getKnownModules(child, s); - } +interface TypstFileMeta { + package: number; + path: string; + isInternal?: boolean; } -function MakeDoc(v: DocElement) { - const knownModules = new Set(); - getKnownModules(v, knownModules); - console.log("MakeDoc", v, knownModules); +interface TypstPackageMeta { + namespace: string; + name: string; + version: string; +} + +function MakeDoc(root: DocElement) { + // s: TypstFileMeta[], + // t: TypstPackageMeta[] + let knownFiles: TypstFileMeta[] = []; + let knownPackages: TypstPackageMeta[] = []; + let selfPackageId = -1; + // module-symbol-module-src.lib-barchart + getKnownPackages(root); + processInternalModules(root); + console.log("MakeDoc", root, knownFiles, knownPackages); + + function getKnownPackages(v: DocElement) { + for (const child of v.children) { + if (child.kind === DocKind.Package) { + knownFiles = [...child.data.pkgEndData["files"]]; + knownFiles.forEach((e) => { + e.path = e.path.replace(/\\/g, "/"); + }); + knownPackages = [...child.data.pkgEndData["packages"]]; + selfPackageId = knownPackages.findIndex( + (e) => + e.namespace === child.data.namespace && + e.name === child.data.name && + e.version === child.data.version + ); + return; + } + getKnownPackages(child); + } + } + + function processInternalModules(v: DocElement) { + if (v.kind === DocKind.Module) { + v.data.aka = v.data.aka || []; + v.data.realAka = v.data.aka.filter((e: string) => !e.includes(".-.")); + knownFiles[v.data.loc].isInternal = v.data.realAka.length === 0; + } + for (const child of v.children) { + processInternalModules(child); + } + } + + function genFileId(file: TypstFileMeta) { + if (!file) { + return "not-found"; + } + const pkg = knownPackages[file.package]; + const pathId = file.path.replaceAll("\\", ".").replaceAll("/", "."); + if (pkg) { + return `module-${pkg.namespace}-${pkg.name}-${pkg.version}-${pathId}`; + } + return `module-${pathId}`; + } + + function getExternalPackage(loc: number) { + if (loc < 0 || loc >= knownFiles.length) { + return undefined; + } + // return knownFiles[loc]?.package !== selfPackageId; + if (knownFiles[loc]?.package === selfPackageId) { + return undefined; + } + return knownPackages[knownFiles[loc]?.package]; + } function Item(v: DocElement): ChildDom { switch (v.kind) { @@ -317,6 +382,18 @@ function MakeDoc(v: DocElement) { } } + function ItemDoc(v: DocElement): ChildDom { + return div({ + style: "margin-left: 0.62em", + innerHTML: v.contents.join(""), + }); + } + + function ShortItemDoc(v: DocElement): ChildDom[] { + console.log("item ref to ", v); + return [ItemDoc(v)]; + } + function ModuleBody(v: DocElement) { const modules = []; const functions = []; @@ -346,8 +423,43 @@ function MakeDoc(v: DocElement) { } } + // sort modules + modules.sort((x, y) => { + const xIsExternal = knownFiles[x.data?.loc[0]].package; + const yIsExternal = knownFiles[y.data?.loc[0]].package; + if (xIsExternal != yIsExternal) { + return xIsExternal ? 1 : -1; + } + + const xIsInternal = knownFiles[x.data?.loc[0]].isInternal; + const yIsInternal = knownFiles[y.data?.loc[0]].isInternal; + if (xIsInternal != yIsInternal) { + return xIsInternal ? 1 : -1; + } + + return x.id.localeCompare(y.id); + }); + const chs = []; + // if (v.data?.aka) { + // const aka: string[] = v.data.aka.filter( + // (e: string) => e && !e.includes(".-.") + // ); + // chs.push( + // ul( + // ...[ + // aka.map((moduleId: string) => + // li( + // `referenced as `, + // a({ href: `#symbol-module-${moduleId}` }, moduleId) + // ) + // ), + // ] + // ) + // ); + // } + if (modules.length > 0) { chs.push(h2("Modules"), div(...modules.map(ModuleRefItem))); } @@ -368,9 +480,30 @@ function MakeDoc(v: DocElement) { } function ModuleItem(v: DocElement) { + const fileLoc = v.data.loc; + const fid = genFileId(knownFiles[fileLoc]); + const isInternal = knownFiles[fileLoc]?.isInternal; + console.log("ModuleItem", v, fid); + + const title = []; + if (isInternal) { + title.push( + span( + { + style: "text-decoration: underline", + title: `It is inaccessible by paths`, + }, + "Module" + ), + code(" ", knownFiles[fileLoc]?.path || v.id) + ); + } else { + title.push(span(`Module: ${v.id}`)); + } + return div( { class: "tinymist-module" }, - h1({ id: `module-${v.id}` }, `Module: ${v.data.prefix}`), + h1({ id: v.id }, ...(fid ? [span({ id: fid }, ...title)] : title)), ModuleBody(v) ); } @@ -397,16 +530,53 @@ function MakeDoc(v: DocElement) { } function ModuleRefItem(v: DocElement) { - const isExternal = !knownModules.has(v.id); + // const isExternal = !v.data.loc; + const fileLoc = v.data.loc; + const extPkg = getExternalPackage(fileLoc?.[0]); + const internal = knownFiles[fileLoc?.[0]].isInternal; let body; - if (isExternal) { - body = code("external ", v.data.name); - } else { + if (extPkg) { body = code( + extPkg.namespace === "preview" + ? a( + { + href: `https://typst.app/universe/package/${extPkg.name}/${extPkg.version}`, + style: "text-decoration: underline", + title: `In external package @${extPkg.namespace}/${extPkg.name}:${extPkg.version}`, + }, + "external" + ) + : span( + { + style: "text-decoration: underline", + title: `In local package @${extPkg.namespace}/${extPkg.name}:${extPkg.version}`, + }, + "external" + ), + code(" ", v.data.name) + ); + } else { + const file = knownFiles[fileLoc?.[0]]; + const fid = genFileId(file); + const bodyPre = internal + ? code( + span( + { + style: "text-decoration: underline", + title: `This module is inaccessible by paths`, + }, + "internal" + ), + code(" ") + ) + : code(); + + body = code( + bodyPre, a( { - href: `#module-${v.id}`, + href: `#${fid}`, }, v.data.name ) @@ -415,6 +585,7 @@ function MakeDoc(v: DocElement) { return div( { + id: v.id, class: "tinymist-module-ref", }, div( @@ -439,7 +610,18 @@ function MakeDoc(v: DocElement) { function FuncItem(v: DocElement) { const sig = v.data.signature; - let funcTitle = [code(v.data.name), "("]; + + const export_again = v.data.export_again + ? [kwHl("external"), code(" ")] + : []; + // symbol-function-src.draw.grouping-place-anchors + const name = a( + { + id: v.id, + }, + code(v.data.name) + ); + let funcTitle = [...export_again, name, "("]; if (sig) { // funcTitle.push(...sig.pos.map((e: DocParam) => code(e.name))); for (let i = 0; i < sig.pos.length; i++) { @@ -473,11 +655,7 @@ function MakeDoc(v: DocElement) { h3({ class: "doc-symbol-name" }, code(...funcTitle)) ), ...SigPreview(v), - div({ - style: "margin-left: 0.62em", - innerHTML: v.contents.join(""), - }), - ...SigDocs(v) + ...(v.data.export_again ? ShortItemDoc(v) : [ItemDoc(v), ...SigDocs(v)]) ); } @@ -534,7 +712,7 @@ function MakeDoc(v: DocElement) { if (parsed_docs?.return_ty || sig.ret_ty) { let paramTitle = [codeHl("op", "-> ")]; - sigTypeHighlighted(parsed_docs.return_ty, sig.ret_ty, paramTitle); + sigTypeHighlighted(parsed_docs?.return_ty, sig.ret_ty, paramTitle); res.push(h3("Resultant")); res.push( @@ -561,8 +739,6 @@ function MakeDoc(v: DocElement) { res.push(h3("Parameters")); } - console.log("SigDocs", { paramsAll, docsMapping }); - for (const { kind, param } of paramsAll) { let docs: string[] = []; const docsMeta = docsMapping.get(param.name); @@ -641,9 +817,24 @@ function MakeDoc(v: DocElement) { // } // return code(param.name); // }), + // http://localhost:5173/#symbol-function-src.lib.draw-copy-anchors + // http://localhost:5173/#symbol-function-src.draw.grouping-copy-anchors + const export_again = v.data.export_again + ? [ + a( + { + href: v.data.external_link, + title: "this symbol is re-exported from other modules", + }, + kwHl("external") + ), + code(" "), + ] + : []; const sigTitle = [ - code(kwHl("let")), + ...export_again, + kwHl("let"), code(" "), code(fnHl(v.data.name)), code("("), @@ -714,14 +905,11 @@ function MakeDoc(v: DocElement) { // ) ) ), - div({ - style: "margin-left: 0.62em", - innerHTML: v.contents.join(""), - }) + ItemDoc(v) ); } - return Item(v); + return Item(root); } function sigTypeHighlighted( @@ -729,7 +917,6 @@ function sigTypeHighlighted( inferred: [string, string] | undefined, target: ChildDom[] ) { - console.log("sigTypeHighlighted", { types, inferred }); if (types) { typeHighlighted(types, target); } else if (inferred) {