docs: improve and add contribution guide for syntaxes (#471)

* dev: reformat metadata of raw languages * docs: improve and add contribution guide for syntaxes
2025-12-23 08:47:50 +00:00 · 2024-07-27 23:04:52 +08:00 · 2024-07-27 23:04:52 +08:00 · 08b9c10aa1
commit 08b9c10aa1
parent 4162b6c65f
6 changed files with 210 additions and 224 deletions
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@ -15,6 +15,11 @@ Tinymist provides a single integrated language service for Typst.

 **Editor Frontends** – Leveraging the interface of LSP, tinymist provides frontends to each editor, located in the [editor folder](./editors).

+## Installing Toolchain
+
+- [Cargo](https://doc.rust-lang.org/cargo/) – Cargo is the Rust package manager.
+- [Yarn](https://yarnpkg.com/) – Yarn is a package manager that doubles down as project manager.
+
 ## Building and Running

 To build tinymist LSP:
@ -31,6 +36,23 @@ cargo build --profile=gh-release

 To run VS Code extension locally, open the repository in VS Code and press `F5` to start a debug session to extension.

+## Local Documentation
+
+To serve the documentation locally, run:
+
+```bash
+yarn docs
+```
+
+To generate and open crate documentation, run:
+
+```bash
+yarn docs:rs --open
+```
+
+> [!Tip]  
+> Check [Shiroa](https://myriad-dreamin.github.io/shiroa/guide/installation.html) to install the `shiroa` command for documentation generation.
+
 ## Server Entries

 - `tinymist probe` – do nothing, which just probes that the binary is working.
@ -50,6 +72,17 @@ cargo insta test -p tinymist-query --accept
 > [!Tip]  
 > Check [Cargo Insta](https://insta.rs/docs/cli/) to learn and install the `insta` command.

+## Running Syntax Grammar Tests
+
+This is required if you are going to change the textmate grammar in `syntaxes/textmate`.
+
+```bash
+# in root
+yarn test:grammar
+# Or in syntaxes/textmate
+cd syntaxes/textmate && yarn test
+```
+
 ## Running E2E Tests

 This is required if you have changed any code in `crates/tinymist` or `crates/tinymist-query`.
--- a/crates/tinymist-assets/src/lib.rs
+++ b/crates/tinymist-assets/src/lib.rs
@ -1,4 +1,4 @@
-/// If this file is not found, please refer to https://enter-tainer.github.io/typst-preview/dev.html to build the frontend.
+/// If this file is not found, please refer to <https://enter-tainer.github.io/typst-preview/dev.html> to build the frontend.
 #[cfg(feature = "typst-preview")]
 pub const TYPST_PREVIEW_HTML: &str = include_str!("typst-preview.html");
 #[cfg(not(feature = "typst-preview"))]
--- a/package.json
+++ b/package.json
@ -15,7 +15,8 @@
    "build:editor-tools": "cd tools/editor-tools/ && yarn run build",
    "build:preview": "cd tools/typst-preview-frontend && yarn run build && rimraf ../../crates/tinymist-assets/src/typst-preview.html && cpr ./dist/index.html ../../crates/tinymist-assets/src/typst-preview.html",
    "docs": "shiroa serve -w . docs/tinymist",
-    "docs:rs": "cargo doc --workspace --no-deps"
+    "docs:rs": "cargo doc --workspace --no-deps",
+    "test:grammar": "cd syntaxes/textmate && yarn run test"
  },
  "dependencies": {},
  "devDependencies": {
--- a/syntaxes/textmate/README.md
+++ b/syntaxes/textmate/README.md
@ -0,0 +1,38 @@
+
+# Syntax Highlighting for Typst
+
+This folder contains the syntax highlighting for Typst. The syntax highlighting is written in the TextMate format.
+
+The syntax highlighting is written in TypeScript, and ensures correct grammar by [./textmate.ts](./textmate.mts).
+
+### Building
+
+The following script running the TypeSCript program will generate the TextMate grammar file:
+
+```shell
+yarn compile
+```
+
+### Testing
+
+```shell
+yarn test
+```
+
+### Register languages for raw highlighting
+
+Goto [fenced.meta.mts](./fenced.meta.mts) and add a line like this:
+
+```json
+{ "candidates": ["erlang"] }
+```
+
+Three possible kinds:
+- `{ candidates: ["someLanguage", ...rests] }` - using textmate parser registered as `source.someLanguage`.
+  - The `rests` of the candidates can also be used as language tag of fenced code blocks.
+- `{ as: "text.xxx", candidates }` - using textmate parser registered as `text.xxx`.
+- `{ as: ["text.xxx", ...restScopes], candidates }` - using textmate parser `text.xxx` first, and `restScopes` parsers in order.
+
+## Contributing
+
+See [CONTRIBUTING.md](https://github.com/Myriad-Dreamin/tinymist/blob/main/CONTRIBUTING.md).
--- a/syntaxes/textmate/fenced.meta.mts
+++ b/syntaxes/textmate/fenced.meta.mts
@ -0,0 +1,63 @@
+import type { Lang } from "./fenced.mjs";
+
+// prettier-ignore
+export const languages: Lang[] = [
+  { candidates: ["batchfile", "bat", "batch"] },
+  { candidates: ["bibtex"] },
+  { candidates: ["c", "h"] },
+  { candidates: ["clojure", "clj", "cljs"] },
+  { candidates: ["coffee", "Cakefile", "coffee.erb"] },
+  { candidates: ["cpp", "c++", "cxx"] },
+  { candidates: ["css", "css.erb"] },
+  { candidates: ["cs", "csharp", "c#"] },
+  { candidates: ["dart"] },
+  { candidates: ["diff", "patch", "rej"] },
+  { candidates: ["dockerfile", "Dockerfile"] },
+  { candidates: ["elixir"] },
+  { candidates: ["erlang"] },
+  { candidates: ["fs", "fsharp", "f#"] },
+  { /* Git Commit */ as: "text.git-commit", candidates: ["git-commit", "COMMIT_EDITMSG", "MERGE_MSG"] },
+  { /* Git Rebase */ as: "text.git-rebase", candidates: ["git-rebase", "git-rebase-todo"] },
+  { candidates: ["go", "golang"] },
+  { candidates: ["groovy", "gvy"] },
+  { /* Handlebars */ as: "text.html.handlebars", candidates: ["handlebars", "hbs"] },
+  { /* HTML */ as: "text.html.basic", candidates: ["html", "htm", "shtml", "xhtml", "inc", "tmpl", "tpl"] },
+  { candidates: ["ini", "conf"] },
+  { candidates: ["java", "bsh"] },
+  { candidates: ["js", "jsx", "javascript", "es6", "mjs", "cjs", "dataviewjs"] },
+  { /* JSON with Comments */ as: "source.json.comments", candidates: ["jsonc"] },
+  { candidates: ["json", "json5", "sublime-settings", "sublime-menu", "sublime-keymap", "sublime-mousemap", "sublime-theme", "sublime-build", "sublime-project", "sublime-completions"] },
+  { candidates: ["julia"] },
+  { /* LaTeX */ as: "text.tex.latex", candidates: ["latex", "tex"] },
+  { /* Less */ as: "source.css.less", candidates: ["less"] },
+  { /* Log */ as: "text.log", candidates: ["log"] },
+  { candidates: ["lua"] },
+  { candidates: ["makefile", "makefile", "GNUmakefile", "OCamlMakefile"] },
+  { /* Markdown */ as: "text.html.markdown", candidates: ["markdown", "md"] },
+  { candidates: ["objc", "objective-c", "mm", "obj-c", "m"] },
+  { candidates: ["perl", "pl", "pm", "pod", "t", "PL", "psgi", "vcl"] },
+  { /* Perl 6 */ as: "source.perl.6", candidates: ["perl6", "p6", "pl6", "pm6", "nqp"] },
+  { /* PHP */ as: ["text.html.basic", "source.php"], candidates: ["php", "php", "php3", "php4", "php5", "phtml", "aw", "ctp"] },
+  { candidates: ["powershell", "ps1", "psm1", "psd1"] },
+  { /* Pug */ as: "text.pug", candidates: ["pug", "jade"] },
+  { candidates: ["r", "R", "r", "s", "S", "Rprofile"] },
+  { /* Regexp (JavaScript) */ as: "source.js.regexp", candidates: ["regexp"] },
+  { /* Regexp (Python) */ as: "source.regexp.python", candidates: ["re"] },
+  { candidates: ["ruby", "rb", "rbx", "rjs", "Rakefile", "rake", "cgi", "fcgi", "gemspec", "irbrc", "Capfile", "ru", "prawn", "Cheffile", "Gemfile", "Guardfile", "Hobofile", "Vagrantfile", "Appraisals", "Rantfile", "Berksfile", "Berksfile.lock", "Thorfile", "Puppetfile"] },
+  { /* SCSS */ as: "source.css.scss", candidates: ["scss"] },
+  { candidates: ["sql", "ddl", "dml"] },
+  { candidates: ["swift"] },
+  { candidates: ["typst", "typ"] },
+  { candidates: ["typst-code", "typc"] },
+  { /* XML */ as: "text.xml", candidates: ["xml", "xsd", "tld", "jsp", "pt", "cpt", "dtml", "rss", "opml"] },
+  { /* XSL */ as: "text.xml.xsl", candidates: ["xsl", "xslt"] },
+  { candidates: ["yaml", "yml"] },
+  { candidates: ["python", "py", "py3", "rpy", "pyw", "cpy", "SConstruct", "Sconstruct", "sconstruct", "SConscript", "gyp", "gypi"] },
+  { candidates: ["rust", "rs"] },
+  { candidates: ["scala", "sbt"] },
+  { candidates: ["shell", "sh", "bash", "zsh", "bashrc", "bash_profile", "bash_login", "profile", "bash_logout", ".textmate_init"] },
+  { candidates: ["ts", "typescript"] },
+  { candidates: ["tsx"] },
+  { candidates: ["twig"] },
+  { /* VB */ as: "source.asp.vb.net", candidates: ["vb"] },
+]
--- a/syntaxes/textmate/fenced.mts
+++ b/syntaxes/textmate/fenced.mts
@ -1,236 +1,87 @@
 import * as textmate from "./textmate.mjs";
+import { languages as rawLanguages } from "./fenced.meta.mjs";

 const IDENTIFIER_BARE = /[\p{XID_Start}_][\p{XID_Continue}_\-]*/u;

-const blockRawLangGen =
-  (ass0: string | undefined, ...ass: string[]) =>
-  (...candidates: string[]): textmate.Pattern => {
-    const lang = candidates[0];
-    const sourcePatterns = [
+export interface Lang {
+  as?: string | string[];
+  candidates: string[];
+}
+
+const genLang = (langMeta: Lang): textmate.Pattern => {
+  const lang = langMeta.candidates[0];
+  let includes = langMeta.as;
+  if (!includes) {
+    includes = [`source.${lang}`];
+  } else if (typeof includes === "string") {
+    includes = [includes];
+  }
+
+  const sourcePatterns = includes.map((include) => ({ include }));
+  const candidates = langMeta.candidates.map((s) =>
+    s.replace(/[.+]/g, (e) => `\\${e}`)
+  );
+
+  const enter = (n: number): textmate.Pattern => ({
+    begin: new RegExp(
+      "(`{" + n.toString() + "})" + `(${candidates.join("|")})\\b`
+    ),
+    beginCaptures: {
+      "1": {
+        name: "punctuation.definition.raw.begin.typst",
+      },
+      "2": {
+        name: "fenced_code.block.language.typst",
+      },
+    },
+    end: /\s*(\1)/,
+    endCaptures: {
+      "1": {
+        name: "punctuation.definition.raw.end.typst",
+      },
+    },
+    patterns: [
      {
-        include: ass0 || `source.${lang}`,
+        begin: /(^|\G)(\s*)/,
+        // end: "(?=`{" + n.toString() + ",})",
+        while: "(^|\\G)(?!\\s*`{" + n.toString() + ",}\\s*)",
+        contentName: `meta.embedded.block.${lang}`,
+        patterns: sourcePatterns,
      },
-      ...ass.map((include) => ({ include })),
-    ];
+    ],
+  });

-    const enter = (n: number): textmate.Pattern => ({
-      begin: new RegExp(
-        "(`{" + n.toString() + "})" + `(${candidates.join("|")})\\b`
-      ),
-      beginCaptures: {
-        "1": {
-          name: "punctuation.definition.raw.begin.typst",
-        },
-        "2": {
-          name: "fenced_code.block.language.typst",
-        },
-      },
-      end: /\s*(\1)/,
-      endCaptures: {
-        "1": {
-          name: "punctuation.definition.raw.end.typst",
-        },
-      },
-      patterns: [
-        {
-          begin: /(^|\G)(\s*)/,
-          // end: "(?=`{" + n.toString() + ",})",
-          while: "(^|\\G)(?!\\s*`{" + n.toString() + ",}\\s*)",
-          contentName: `meta.embedded.block.${lang}`,
-          patterns: sourcePatterns,
-        },
-      ],
-    });
-
-    return {
-      name: `markup.raw.block.${lang}`,
-      patterns: [
-        // one line case
-        {
-          match: new RegExp(
-            /(`{3,})/.source +
-              `(${candidates.join("|")})` +
-              /\b(.*?)(\1)/.source
-          ),
-          captures: {
-            "1": {
-              name: "punctuation.definition.raw.begin.typst",
-            },
-            "2": {
-              name: "fenced_code.block.language.typst",
-            },
-            "3": {
-              name: `meta.embedded.block.${lang}`,
-              patterns: sourcePatterns,
-            },
-            "4": {
-              name: "punctuation.definition.raw.end.typst",
-            },
+  return {
+    name: `markup.raw.block.${lang}`,
+    patterns: [
+      // one line case
+      {
+        match: new RegExp(
+          /(`{3,})/.source + `(${candidates.join("|")})` + /\b(.*?)(\1)/.source
+        ),
+        captures: {
+          "1": {
+            name: "punctuation.definition.raw.begin.typst",
+          },
+          "2": {
+            name: "fenced_code.block.language.typst",
+          },
+          "3": {
+            name: `meta.embedded.block.${lang}`,
+            patterns: sourcePatterns,
+          },
+          "4": {
+            name: "punctuation.definition.raw.end.typst",
          },
        },
-        ...[6, 5, 4, 3].map(enter),
-      ],
-    };
+      },
+      ...[6, 5, 4, 3].map(enter),
+    ],
  };
+};

-const blockRawLangAs = (as?: string) => blockRawLangGen(as);
-const blockRawLang = blockRawLangAs();
-
-const ENABLE_RAW_RENDERING = true;
-
-const blockRawLangs_ = [
-  blockRawLang("typst", "typ"),
-  blockRawLang("typst-code", "typc"),
-  blockRawLang("css", "css.erb"),
-  blockRawLangAs("text.html.basic")(
-    "html",
-    "htm",
-    "shtml",
-    "xhtml",
-    "inc",
-    "tmpl",
-    "tpl"
-  ),
-  blockRawLang("ini", "conf"),
-  blockRawLang("java", "bsh"),
-  blockRawLang("lua"),
-  blockRawLang("makefile", "makefile", "GNUmakefile", "OCamlMakefile"),
-  blockRawLang("perl", "pl", "pm", "pod", "t", "PL", "psgi", "vcl"),
-  blockRawLang("r", "R", "r", "s", "S", "Rprofile"),
-  blockRawLang(
-    "ruby",
-    "rb",
-    "rbx",
-    "rjs",
-    "Rakefile",
-    "rake",
-    "cgi",
-    "fcgi",
-    "gemspec",
-    "irbrc",
-    "Capfile",
-    "ru",
-    "prawn",
-    "Cheffile",
-    "Gemfile",
-    "Guardfile",
-    "Hobofile",
-    "Vagrantfile",
-    "Appraisals",
-    "Rantfile",
-    "Berksfile",
-    "Berksfile.lock",
-    "Thorfile",
-    "Puppetfile"
-  ),
-  blockRawLangGen("text.html.basic", "source.php")(
-    "php",
-    "php",
-    "php3",
-    "php4",
-    "php5",
-    "phpt",
-    "phtml",
-    "aw",
-    "ctp"
-  ),
-  blockRawLang("sql", "ddl", "dml"),
-  blockRawLangAs("source.asp.vb.net")("vb"),
-  blockRawLangAs("text.xml")(
-    "xml",
-    "xsd",
-    "tld",
-    "jsp",
-    "pt",
-    "cpt",
-    "dtml",
-    "rss",
-    "opml"
-  ),
-  blockRawLangAs("text.xml.xsl")("xsl", "xslt"),
-  blockRawLang("yaml", "yml"),
-  blockRawLang("batchfile", "bat", "batch"),
-  blockRawLang("clojure", "clj", "cljs"),
-  blockRawLang("coffee", "Cakefile", "coffee.erb"),
-  blockRawLang("c", "h"),
-  blockRawLang("cpp", "c\\+\\+", "cxx"),
-  blockRawLang("diff", "patch", "rej"),
-  blockRawLang("dockerfile", "Dockerfile"),
-  blockRawLangAs("text.git-commit")(
-    "git-commit",
-    "COMMIT_EDITMSG",
-    "MERGE_MSG"
-  ),
-  blockRawLangAs("text.git-rebase")("git-rebase", "git-rebase-todo"),
-  blockRawLang("go", "golang"),
-  blockRawLang("groovy", "gvy"),
-  blockRawLangAs("text.pug")("pug", "jade"),
-  blockRawLang("js", "jsx", "javascript", "es6", "mjs", "cjs", "dataviewjs"),
-  blockRawLangAs("source.js.regexp")("regexp"),
-  blockRawLang(
-    "json",
-    "json5",
-    "sublime-settings",
-    "sublime-menu",
-    "sublime-keymap",
-    "sublime-mousemap",
-    "sublime-theme",
-    "sublime-build",
-    "sublime-project",
-    "sublime-completions"
-  ),
-  blockRawLangAs("source.json.comments")("jsonc"),
-  blockRawLangAs("source.css.less")("less"),
-  blockRawLang("objc", "objective-c", "mm", "obj-c", "m", "h"),
-  blockRawLang("swift"),
-  blockRawLangAs("source.css.scss")("scss"),
-  blockRawLangAs("source.perl.6")("perl6", "p6", "pl6", "pm6", "nqp"),
-  blockRawLang("powershell", "ps1", "psm1", "psd1"),
-  blockRawLang(
-    "python",
-    "py",
-    "py3",
-    "rpy",
-    "pyw",
-    "cpy",
-    "SConstruct",
-    "Sconstruct",
-    "sconstruct",
-    "SConscript",
-    "gyp",
-    "gypi"
-  ),
-  blockRawLang("julia"),
-  blockRawLangAs("source.regexp.python")("re"),
-  blockRawLang("rust", "rs"),
-  blockRawLang("scala", "sbt"),
-  blockRawLang(
-    "shell",
-    "sh",
-    "bash",
-    "zsh",
-    "bashrc",
-    "bash_profile",
-    "bash_login",
-    "profile",
-    "bash_logout",
-    ".textmate_init"
-  ),
-  blockRawLang("ts", "typescript"),
-  blockRawLang("tsx"),
-  blockRawLang("cs", "csharp", "c#"),
-  blockRawLang("fs", "fsharp", "f#"),
-  blockRawLang("dart"),
-  blockRawLangAs("text.html.handlebars")("handlebars", "hbs"),
-  blockRawLangAs("text.html.markdown")("markdown", "md"),
-  blockRawLangAs("text.log")("log"),
-  blockRawLang("erlang"),
-  blockRawLang("elixir"),
-  blockRawLangAs("text.tex.latex")("latex", "tex"),
-  blockRawLangAs("text.bibtex")("bibtex"),
-  blockRawLang("twig"),
-];
-export const blockRawLangs = ENABLE_RAW_RENDERING ? blockRawLangs_ : [];
+const RENDER_LANGS = true;
+export const blockRawLangs = RENDER_LANGS ? rawLanguages.map(genLang) : [];

 export const inlineRaw: textmate.Pattern = {
  name: "markup.raw.inline.typst",