docs: documenting textmate grammar (#1131)

syntaxes/textmate/README.md

@@ -1,9 +1,9 @@
 
 # Syntax Highlighting for Typst
 
-This folder contains the syntax highlighting for Typst. The syntax highlighting is written in the TextMate format.
+This folder contains the syntax highlighting in the TextMate format for Typst.
 
-The syntax highlighting is written in TypeScript, and ensures correct grammar by [./textmate.ts](./textmate.mts).
+To tackle challenge of making the complex grammar for typst markup, the grammar is described by neither JSON nor YAML, but a TypeScript generator program, the [./main.ts](./main.mts). TypeScript ensures correct grammar by static and strong types from [./textmate.ts](./textmate.mts).
 
 ### Building
 
@@ -16,7 +16,12 @@ yarn compile
 ### Testing
 
 ```shell
+// Run unit tests
 yarn test
+// Test on typst/typst
+yarn test:official
+// Test on typst/packages
+yarn test:packages
 ```
 
 ### Register languages for raw highlighting
@@ -33,6 +38,20 @@ Three possible kinds:
 - `{ as: "text.xxx", candidates }` - using textmate parser registered as `text.xxx`.
 - `{ as: ["text.xxx", ...restScopes], candidates }` - using textmate parser `text.xxx` first, and `restScopes` parsers in order.
 
+### GitHub Integration
+
+A variant satisfying GitHub's requirement is managed on [Typst Grammar Repo](https://github.com/michidk/typst-grammar). You can check which version the repository is using by checking the [`build-ref.md`](https://github.com/michidk/typst-grammar/blob/main/build-ref.md) or [`build-ref.json`](https://github.com/michidk/typst-grammar/blob/main/build-ref.json).
+
+The grammar is built by the [build branch's CI.](https://github.com/Myriad-Dreamin/typst-grammar/tree/build)
+
+The grammar is tested continuously by the [main branch's CI.](https://github.com/michidk/typst-grammar/blob/main/.github/workflows/ci.yml) Specifically, it is tested by the command in the CI script:
+
+```bash
+script/grammar-compiler add vendor/grammars/typst-grammar
+```
+
+You can setup your owned environment according to [github-linguist's CONTRIBUTING.md](https://github.com/github-linguist/linguist) to develop the variant locally.
+
 ## Contributing
 
 See [CONTRIBUTING.md](https://github.com/Myriad-Dreamin/tinymist/blob/main/CONTRIBUTING.md).

syntaxes/textmate/tests/unit/math/raw.typ

@@ -0,0 +1 @@
+$ `a` $

syntaxes/textmate/textmate.mts

@@ -1,3 +1,5 @@
+import { ANNOTATE_META } from "./feature.mjs";
+
 export interface Grammar {
   patterns?: Pattern[];
   repository?: Repository;
@@ -151,6 +153,58 @@ export interface Capture {
   patterns?: Pattern[];
 }
 
+/**
+ * A function to create a look-ahead regular expression.
+ *
+ * @param pattern A regular expression
+ * @returns A regular expression that only looks ahead the pattern without consuming it.
+ */
+export function lookAhead(pattern: RegExp) {
+  return new RegExp(`(?=(?:${pattern.source}))`);
+}
+
+/**
+ * Unions a list of regular expressions into a single regular expression that matches any of the given patterns.
+ *
+ * @param patterns A list of regular expressions
+ * @returns A regular expression that matches any of the patterns.
+ */
+export function oneOf(...patterns: RegExp[]) {
+  return new RegExp(
+    patterns
+      .map((p) => {
+        const src = p.source;
+        if (src.startsWith("(")) {
+          return src;
+        }
+
+        return `(?:${src})`;
+      })
+      .join("|")
+  );
+}
+
+/**
+ * A function to replace a group in a regular expression with another regular expression by simple string substitution.
+ *
+ * @param pat A regular expression
+ * @param group the substring to be replaced
+ * @param replacement A regular expression to replace the group
+ */
+export function replaceGroup(pat: RegExp, group: string, replacement: RegExp) {
+  return new RegExp(pat.source.replace(group, replacement.source), pat.flags);
+}
+
+/**
+ * A wrapper function to annotate the scope name.
+ *
+ * @param name A scope name
+ * @returns return the scope name if we should annotate the scope name, otherwise return undefined
+ */
+export function metaName(name: string) {
+  return ANNOTATE_META ? name : undefined;
+}
+
 export function compile(s: Grammar): string {
   return JSON.stringify(
     s,