language-servers/tinymist
1
0
Fork 0
mirror of https://github.com/Myriad-Dreamin/tinymist.git synced 2025-11-20 03:49:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

feat: warning collector and logging for diagnostics in typlite (#2180)
Some checks are pending
tinymist::auto_tag / auto-tag (push) Waiting to run
Details
tinymist::ci / Duplicate Actions Detection (push) Waiting to run
Details
tinymist::ci / Check Clippy, Formatting, Completion, Documentation, and Tests (Linux) (push) Waiting to run
Details
tinymist::ci / Check Minimum Rust version and Tests (Windows) (push) Waiting to run
Details
tinymist::ci / prepare-build (push) Waiting to run
Details
tinymist::ci / announce (push) Blocked by required conditions
Details
tinymist::ci / build (push) Blocked by required conditions
Details
tinymist::gh_pages / build-gh-pages (push) Waiting to run
Details

Browse source 
This PR depends on #2173 and should be merged after it.
This commit is contained in:
Hong Jiarong 2025-10-16 18:18:57 +08:00 • committed by GitHub
parent 29a10c144e
commit cee5bfa4e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 288 additions and 55 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -4982,8 +4982,10 @@ dependencies = [
"comemo", "comemo",
"docx-rs", "docx-rs",
"ecow", "ecow",
"env_logger",
"image", "image",
"insta", "insta",
"log",
"regex", "regex",
"resvg", "resvg",
"tinymist-derive", "tinymist-derive",

6
crates/tinymist-query/src/docs/convert.rs
View file

@ -47,7 +47,7 @@ pub(crate) fn convert_docs(
)); ));
imports.join("; ") imports.join("; ")
}); });
let feat = TypliteFeat { let mut feat = TypliteFeat {
color_theme: Some(ctx.analysis.color_theme), color_theme: Some(ctx.analysis.color_theme),
annotate_elem: true, annotate_elem: true,
soft_error: true, soft_error: true,
@ -67,10 +67,12 @@ pub(crate) fn convert_docs(
w.map_shadow_by_id(w.main(), Bytes::from_string(content.to_owned()))?; w.map_shadow_by_id(w.main(), Bytes::from_string(content.to_owned()))?;
// todo: bad performance // todo: bad performance
w.take_db(); w.take_db();
let w = feat let (w, wrap_info) = feat
.prepare_world(&w, Format::Md) .prepare_world(&w, Format::Md)
.map_err(|e| eco_format!("failed to prepare world: {e}"))?; .map_err(|e| eco_format!("failed to prepare world: {e}"))?;
feat.wrap_info = wrap_info;
let w = Arc::new(w); let w = Arc::new(w);
let res = typlite::Typlite::new(w.clone()) let res = typlite::Typlite::new(w.clone())
.with_feature(feat) .with_feature(feat)

4
crates/typlite/Cargo.toml
View file

@ -34,6 +34,8 @@ typst.workspace = true
typst-svg.workspace = true typst-svg.workspace = true
typst-syntax.workspace = true typst-syntax.workspace = true
typst-html.workspace = true typst-html.workspace = true
log.workspace = true
env_logger = { workspace = true, optional = true }
# Feature: docx # Feature: docx
docx-rs = { workspace = true, optional = true } docx-rs = { workspace = true, optional = true }
@ -51,7 +53,7 @@ clap = ["dep:clap"]
# Note: this is the feature for typlite as a CLI, not for others. # Note: this is the feature for typlite as a CLI, not for others.
# `docx` is enabled in CLI mode, but not in library mode. # `docx` is enabled in CLI mode, but not in library mode.
# `fonts` is enabled in CLI mode. # `fonts` is enabled in CLI mode.
cli = ["clap", "clap/wrap_help", "docx", "fonts", "system"] cli = ["clap", "clap/wrap_help", "docx", "env_logger", "fonts", "system"]
no-content-hint = ["tinymist-project/no-content-hint"] no-content-hint = ["tinymist-project/no-content-hint"]
docx = ["docx-rs", "image", "resvg"] docx = ["docx-rs", "image", "resvg"]

49
crates/typlite/src/diagnostics.rs Normal file
View file

@ -0,0 +1,49 @@
use std::sync::{Arc, Mutex};
use log::warn;
use tinymist_project::diag::print_diagnostics_to_string;
use tinymist_project::{DiagnosticFormat, SourceWorld};
use typst::diag::SourceDiagnostic;
/// Shared collector for Typst warnings emitted during conversion.
#[derive(Clone, Default)]
pub(crate) struct WarningCollector {
inner: Arc<Mutex<Vec<SourceDiagnostic>>>,
}
impl WarningCollector {
/// Extend the collector with multiple warnings.
pub fn extend<I>(&self, warnings: I)
where
I: IntoIterator<Item = SourceDiagnostic>,
{
let mut guard = self.inner.lock().expect("warning collector poisoned");
guard.extend(warnings);
}
/// Clone all collected warnings into a standalone vector.
pub fn snapshot(&self) -> Vec<SourceDiagnostic> {
let guard = self.inner.lock().expect("warning collector poisoned");
guard.clone()
}
}
/// Render warnings into a human-readable string for the CLI.
#[allow(dead_code)]
pub(crate) fn render_warnings<'a>(
world: &dyn SourceWorld,
warnings: impl IntoIterator<Item = &'a SourceDiagnostic>,
) -> Option<String> {
let warnings: Vec<&SourceDiagnostic> = warnings.into_iter().collect();
if warnings.is_empty() {
return None;
}
match print_diagnostics_to_string(world, warnings.into_iter(), DiagnosticFormat::Human) {
Ok(message) => Some(message.to_string()),
Err(err) => {
warn!("failed to render Typst warnings: {err}");
None
}
}
}

2
crates/typlite/src/fixtures/docs/snaps/convert_docs@nest_list.typ.snap
View file

@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/docs/nest_list.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><p>These again are dictionaries with the keys</p><ul><li><span><m1raw lang="" block="false" text="description"></m1raw></span> (optional): The description for the argument.</li></ul><m1parbreak></m1parbreak><p>See show-module() for outputting the results of this function.</p><m1parbreak></m1parbreak><ul><li>name (string): The name for the module.</li><li><p>label-prefix (auto, string): The label-prefix for internal function references. If <span><m1raw lang="" block="false" text="auto"></m1raw></span>, the label-prefix name will be the module name.</p><ul><li>nested something</li><li>nested something 2</li></ul></li></ul><p>-> string</p></m1document></body> <body><m1document><p>These again are dictionaries with the keys</p><ul><li><span><m1raw lang="" block="false" text="description"><code>description</code></m1raw></span> (optional): The description for the argument.</li></ul><m1parbreak></m1parbreak><p>See show-module() for outputting the results of this function.</p><m1parbreak></m1parbreak><ul><li>name (string): The name for the module.</li><li><p>label-prefix (auto, string): The label-prefix for internal function references. If <span><m1raw lang="" block="false" text="auto"><code>auto</code></m1raw></span>, the label-prefix name will be the module name.</p><ul><li>nested something</li><li>nested something 2</li></ul></li></ul><p>-> string</p></m1document></body>
</html> </html>
===== =====

3
crates/typlite/src/fixtures/docs/snaps/convert_docs@tidy.typ.snap
View file

@ -2,7 +2,6 @@
source: crates/typlite/src/tests.rs source: crates/typlite/src/tests.rs
expression: "conv(world, ConvKind::Md { for_docs: true })" expression: "conv(world, ConvKind::Md { for_docs: true })"
input_file: crates/typlite/src/fixtures/docs/tidy.typ input_file: crates/typlite/src/fixtures/docs/tidy.typ
snapshot_kind: text
--- ---
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
@ -10,7 +9,7 @@ snapshot_kind: text
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><p>These again are dictionaries with the keys</p><ul><li><span><m1raw lang="" block="false" text="description"></m1raw></span> (optional): The description for the argument.</li><li><span><m1raw lang="" block="false" text="types"></m1raw></span> (optional): A list of accepted argument types.</li><li><span><m1raw lang="" block="false" text="default"></m1raw></span> (optional): Default value for this argument.</li></ul><m1parbreak></m1parbreak><p>See show-module() for outputting the results of this function.</p><m1parbreak></m1parbreak><ul><li>content (string): Content of <span><m1raw lang="" block="false" text=".typ"></m1raw></span> file to analyze for docstrings.</li><li>name (string): The name for the module.</li><li>label-prefix (auto, string): The label-prefix for internal function references. If <span><m1raw lang="" block="false" text="auto"></m1raw></span>, the label-prefix name will be the module name.</li><li>require-all-parameters (boolean): Require that all parameters of a functions are documented and fail if some are not.</li><li>scope (dictionary): A dictionary of definitions that are then available in all function and parameter descriptions.</li><li>preamble (string): Code to prepend to all code snippets shown with <span><m1raw lang="" block="false" text="#example()"></m1raw></span>. This can for instance be used to import something from the scope.</li></ul><p>-> string</p></m1document></body> <body><m1document><p>These again are dictionaries with the keys</p><ul><li><span><m1raw lang="" block="false" text="description"><code>description</code></m1raw></span> (optional): The description for the argument.</li><li><span><m1raw lang="" block="false" text="types"><code>types</code></m1raw></span> (optional): A list of accepted argument types.</li><li><span><m1raw lang="" block="false" text="default"><code>default</code></m1raw></span> (optional): Default value for this argument.</li></ul><m1parbreak></m1parbreak><p>See show-module() for outputting the results of this function.</p><m1parbreak></m1parbreak><ul><li>content (string): Content of <span><m1raw lang="" block="false" text=".typ"><code>.typ</code></m1raw></span> file to analyze for docstrings.</li><li>name (string): The name for the module.</li><li>label-prefix (auto, string): The label-prefix for internal function references. If <span><m1raw lang="" block="false" text="auto"><code>auto</code></m1raw></span>, the label-prefix name will be the module name.</li><li>require-all-parameters (boolean): Require that all parameters of a functions are documented and fail if some are not.</li><li>scope (dictionary): A dictionary of definitions that are then available in all function and parameter descriptions.</li><li>preamble (string): Code to prepend to all code snippets shown with <span><m1raw lang="" block="false" text="#example()"><code>#example()</code></m1raw></span>. This can for instance be used to import something from the scope.</li></ul><p>-> string</p></m1document></body>
</html> </html>
===== =====

2
crates/typlite/src/fixtures/integration/snaps/convert@figure_raw.typ.snap
View file

@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/integration/figure_raw.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><m1figure caption=""><m1raw lang="md" block="true" text="markdown"></m1raw></m1figure></m1document></body> <body><m1document><m1figure caption=""><m1raw lang="md" block="true" text="markdown"><pre>markdown</pre></m1raw></m1figure></m1document></body>
</html> </html>
===== =====

19
crates/typlite/src/fixtures/integration/snaps/convert@issue-1845.typ.snap
View file

@ -10,11 +10,10 @@ input_file: crates/typlite/src/fixtures/integration/issue-1845.typ
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><m1grid><m1table><table><tr><td>Header</td><td>Row</td></tr><tr><td><m1raw lang="" block="true" text="Code line 1 <body><m1document><m1grid><m1table><table><tr><td>Header</td><td>Row</td></tr><tr><td><m1raw lang="" block="true" text="Code line 1
Code line 2"></m1raw></td><td>Regular text</td></tr></table></m1table></m1grid></m1document></body> Code line 2"><pre><p>Code line 1</p><m1linebreak></m1linebreak><p>Code line 2</p></pre></m1raw></td><td>Regular text</td></tr></table></m1table></m1grid></m1document></body>
</html> </html>
===== =====
<!-- typlite warning: block content detected inside table cell; exported original HTML table -->
<table> <table>
<tr> <tr>
@ -36,7 +35,23 @@ Row
<m1raw lang="" block="true" text="Code line 1 <m1raw lang="" block="true" text="Code line 1
Code line 2"> Code line 2">
<pre>
<p>
Code line 1
</p><m1linebreak>
</m1linebreak><p>
Code line 2
</p>
</pre>
</m1raw> </m1raw>

4
crates/typlite/src/fixtures/integration/snaps/convert@outline.typ.snap
View file

@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/integration/outline.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><m1outline><m1heading level="1"><span style="display: inline-block;">Contents</span></m1heading><m1outentry level="2"><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading></m1outentry><m1outentry level="3"><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading></m1outentry></m1outline><m1parbreak></m1parbreak><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading><m1parbreak></m1parbreak><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading><m1parbreak></m1parbreak><p><span><m1link dest="https://example.com">This is a link to example.com</m1link></span></p><m1parbreak></m1parbreak><p>Inline <span><m1raw lang="" block="false" text="code"></m1raw></span> has <span><m1raw lang="" block="false" text="back-ticks around"></m1raw></span> it.</p><m1parbreak></m1parbreak><m1raw lang="cs" block="true" text="using System.IO.Compression; <body><m1document><m1outline><m1heading level="1"><span style="display: inline-block;">Contents</span></m1heading><m1outentry level="2"><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading></m1outentry><m1outentry level="3"><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading></m1outentry></m1outline><m1parbreak></m1parbreak><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading><m1parbreak></m1parbreak><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading><m1parbreak></m1parbreak><p><span><m1link dest="https://example.com">This is a link to example.com</m1link></span></p><m1parbreak></m1parbreak><p>Inline <span><m1raw lang="" block="false" text="code"><code>code</code></m1raw></span> has <span><m1raw lang="" block="false" text="back-ticks around"><code>back-ticks around</code></m1raw></span> it.</p><m1parbreak></m1parbreak><m1raw lang="cs" block="true" text="using System.IO.Compression;
#pragma warning disable 414, 3021 #pragma warning disable 414, 3021
@ -24,7 +24,7 @@ namespace MyApplication
return new List<int>(new int[] { 1, 2, 3 }) return new List<int>(new int[] { 1, 2, 3 })
} }
} }
}"></m1raw><m1parbreak></m1parbreak><p>Math inline:</p><m1eqinline>redacted-frame</m1eqinline><p>and block:</p><m1eqblock>redacted-frame</m1eqblock><m1parbreak></m1parbreak><ul><li>First item</li><li><p>Second item</p><ol><li>First sub-item</li><li><p>Second sub-item</p><ul><li>First sub-sub-item</li></ul></li></ol></li></ul><m1parbreak></m1parbreak><dl><dt>First term</dt><dd>First definition</dd></dl><m1parbreak></m1parbreak><m1table><table><tr><td>0</td><td>1</td><td>2</td><td>3</td><td>4</td><td>5</td><td>6</td><td>7</td><td>8</td><td>9</td><td>10</td><td>11</td><td>12</td><td>13</td><td>14</td><td>15</td><td>16</td><td>17</td><td>18</td><td>19</td></tr></table></m1table></m1document></body> }"><pre><p>using System.IO.Compression;</p><m1linebreak></m1linebreak><m1linebreak></m1linebreak><p>#pragma warning disable 414, 3021</p><m1linebreak></m1linebreak><m1linebreak></m1linebreak><p>namespace MyApplication</p><m1linebreak></m1linebreak><p>{</p><m1linebreak></m1linebreak><p> [Obsolete("...")]</p><m1linebreak></m1linebreak><p> class Program : IInterface</p><m1linebreak></m1linebreak><p> {</p><m1linebreak></m1linebreak><p> public static List&lt;int> JustDoIt(int count)</p><m1linebreak></m1linebreak><p> {</p><m1linebreak></m1linebreak><p> Console.WriteLine($"Hello {Name}!");</p><m1linebreak></m1linebreak><p> return new List&lt;int>(new int[] { 1, 2, 3 })</p><m1linebreak></m1linebreak><p> }</p><m1linebreak></m1linebreak><p> }</p><m1linebreak></m1linebreak><p>}</p></pre></m1raw><m1parbreak></m1parbreak><p>Math inline:</p><m1eqinline>redacted-frame</m1eqinline><p>and block:</p><m1eqblock>redacted-frame</m1eqblock><m1parbreak></m1parbreak><ul><li>First item</li><li><p>Second item</p><ol><li>First sub-item</li><li><p>Second sub-item</p><ul><li>First sub-sub-item</li></ul></li></ol></li></ul><m1parbreak></m1parbreak><dl><dt>First term</dt><dd>First definition</dd></dl><m1parbreak></m1parbreak><m1table><table><tr><td>0</td><td>1</td><td>2</td><td>3</td><td>4</td><td>5</td><td>6</td><td>7</td><td>8</td><td>9</td><td>10</td><td>11</td><td>12</td><td>13</td><td>14</td><td>15</td><td>16</td><td>17</td><td>18</td><td>19</td></tr></table></m1table></m1document></body>
</html> </html>
===== =====

4
crates/typlite/src/fixtures/integration/snaps/convert@raw_inline.typ.snap
View file

@ -1,6 +1,6 @@
--- ---
source: crates/typlite/src/tests.rs source: crates/typlite/src/tests.rs
expression: "conv(world, false)" expression: "conv(world, ConvKind::Md { for_docs: false })"
input_file: crates/typlite/src/fixtures/integration/raw_inline.typ input_file: crates/typlite/src/fixtures/integration/raw_inline.typ
--- ---
<!DOCTYPE html> <!DOCTYPE html>
@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/integration/raw_inline.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document>Some inlined raw <span><m1raw lang="" block="false" text="a"></m1raw></span>, <span><m1raw lang="c" block="false" text="b"></m1raw></span></m1document></body> <body><m1document>Some inlined raw <span><m1raw lang="" block="false" text="a"><code>a</code></m1raw></span>, <span><m1raw lang="c" block="false" text="b"><code>b</code></m1raw></span></m1document></body>
</html> </html>
===== =====

2
crates/typlite/src/fixtures/integration/snaps/convert_tex@figure_raw.typ.snap
View file

@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/integration/figure_raw.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><m1figure caption=""><m1raw lang="md" block="true" text="markdown"></m1raw></m1figure></m1document></body> <body><m1document><m1figure caption=""><m1raw lang="md" block="true" text="markdown"><pre>markdown</pre></m1raw></m1figure></m1document></body>
</html> </html>
===== =====

2
crates/typlite/src/fixtures/integration/snaps/convert_tex@issue-1845.typ.snap
View file

@ -10,7 +10,7 @@ input_file: crates/typlite/src/fixtures/integration/issue-1845.typ
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><m1grid><m1table><table><tr><td>Header</td><td>Row</td></tr><tr><td><m1raw lang="" block="true" text="Code line 1 <body><m1document><m1grid><m1table><table><tr><td>Header</td><td>Row</td></tr><tr><td><m1raw lang="" block="true" text="Code line 1
Code line 2"></m1raw></td><td>Regular text</td></tr></table></m1table></m1grid></m1document></body> Code line 2"><pre><p>Code line 1</p><m1linebreak></m1linebreak><p>Code line 2</p></pre></m1raw></td><td>Regular text</td></tr></table></m1table></m1grid></m1document></body>
</html> </html>
===== =====

4
crates/typlite/src/fixtures/integration/snaps/convert_tex@outline.typ.snap
View file

@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/integration/outline.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document><m1outline><m1heading level="1"><span style="display: inline-block;">Contents</span></m1heading><m1outentry level="2"><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading></m1outentry><m1outentry level="3"><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading></m1outentry></m1outline><m1parbreak></m1parbreak><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading><m1parbreak></m1parbreak><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading><m1parbreak></m1parbreak><p><span><m1link dest="https://example.com">This is a link to example.com</m1link></span></p><m1parbreak></m1parbreak><p>Inline <span><m1raw lang="" block="false" text="code"></m1raw></span> has <span><m1raw lang="" block="false" text="back-ticks around"></m1raw></span> it.</p><m1parbreak></m1parbreak><m1raw lang="cs" block="true" text="using System.IO.Compression; <body><m1document><m1outline><m1heading level="1"><span style="display: inline-block;">Contents</span></m1heading><m1outentry level="2"><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading></m1outentry><m1outentry level="3"><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading></m1outentry></m1outline><m1parbreak></m1parbreak><m1heading level="2"><span style="display: inline-block;">Heading 1</span></m1heading><m1parbreak></m1parbreak><m1heading level="3"><span style="display: inline-block;">Heading 2</span></m1heading><m1parbreak></m1parbreak><p><span><m1link dest="https://example.com">This is a link to example.com</m1link></span></p><m1parbreak></m1parbreak><p>Inline <span><m1raw lang="" block="false" text="code"><code>code</code></m1raw></span> has <span><m1raw lang="" block="false" text="back-ticks around"><code>back-ticks around</code></m1raw></span> it.</p><m1parbreak></m1parbreak><m1raw lang="cs" block="true" text="using System.IO.Compression;
#pragma warning disable 414, 3021 #pragma warning disable 414, 3021
@ -24,7 +24,7 @@ namespace MyApplication
return new List<int>(new int[] { 1, 2, 3 }) return new List<int>(new int[] { 1, 2, 3 })
} }
} }
}"></m1raw><m1parbreak></m1parbreak><p>Math inline:</p><m1eqinline>redacted-frame</m1eqinline><p>and block:</p><m1eqblock>redacted-frame</m1eqblock><m1parbreak></m1parbreak><ul><li>First item</li><li><p>Second item</p><ol><li>First sub-item</li><li><p>Second sub-item</p><ul><li>First sub-sub-item</li></ul></li></ol></li></ul><m1parbreak></m1parbreak><dl><dt>First term</dt><dd>First definition</dd></dl><m1parbreak></m1parbreak><m1table><table><tr><td>0</td><td>1</td><td>2</td><td>3</td><td>4</td><td>5</td><td>6</td><td>7</td><td>8</td><td>9</td><td>10</td><td>11</td><td>12</td><td>13</td><td>14</td><td>15</td><td>16</td><td>17</td><td>18</td><td>19</td></tr></table></m1table></m1document></body> }"><pre><p>using System.IO.Compression;</p><m1linebreak></m1linebreak><m1linebreak></m1linebreak><p>#pragma warning disable 414, 3021</p><m1linebreak></m1linebreak><m1linebreak></m1linebreak><p>namespace MyApplication</p><m1linebreak></m1linebreak><p>{</p><m1linebreak></m1linebreak><p> [Obsolete("...")]</p><m1linebreak></m1linebreak><p> class Program : IInterface</p><m1linebreak></m1linebreak><p> {</p><m1linebreak></m1linebreak><p> public static List&lt;int> JustDoIt(int count)</p><m1linebreak></m1linebreak><p> {</p><m1linebreak></m1linebreak><p> Console.WriteLine($"Hello {Name}!");</p><m1linebreak></m1linebreak><p> return new List&lt;int>(new int[] { 1, 2, 3 })</p><m1linebreak></m1linebreak><p> }</p><m1linebreak></m1linebreak><p> }</p><m1linebreak></m1linebreak><p>}</p></pre></m1raw><m1parbreak></m1parbreak><p>Math inline:</p><m1eqinline>redacted-frame</m1eqinline><p>and block:</p><m1eqblock>redacted-frame</m1eqblock><m1parbreak></m1parbreak><ul><li>First item</li><li><p>Second item</p><ol><li>First sub-item</li><li><p>Second sub-item</p><ul><li>First sub-sub-item</li></ul></li></ol></li></ul><m1parbreak></m1parbreak><dl><dt>First term</dt><dd>First definition</dd></dl><m1parbreak></m1parbreak><m1table><table><tr><td>0</td><td>1</td><td>2</td><td>3</td><td>4</td><td>5</td><td>6</td><td>7</td><td>8</td><td>9</td><td>10</td><td>11</td><td>12</td><td>13</td><td>14</td><td>15</td><td>16</td><td>17</td><td>18</td><td>19</td></tr></table></m1table></m1document></body>
</html> </html>
===== =====

2
crates/typlite/src/fixtures/integration/snaps/convert_tex@raw_inline.typ.snap
View file

@ -9,7 +9,7 @@ input_file: crates/typlite/src/fixtures/integration/raw_inline.typ
<meta charset="utf-8"> <meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1"> <meta name="viewport" content="width=device-width, initial-scale=1">
</head> </head>
<body><m1document>Some inlined raw <span><m1raw lang="" block="false" text="a"></m1raw></span>, <span><m1raw lang="c" block="false" text="b"></m1raw></span></m1document></body> <body><m1document>Some inlined raw <span><m1raw lang="" block="false" text="a"><code>a</code></m1raw></span>, <span><m1raw lang="c" block="false" text="b"><code>b</code></m1raw></span></m1document></body>
</html> </html>
===== =====

151
crates/typlite/src/lib.rs
View file

@ -5,6 +5,7 @@
pub mod attributes; pub mod attributes;
pub mod common; pub mod common;
mod diagnostics;
mod error; mod error;
pub mod parser; pub mod parser;
pub mod tags; pub mod tags;
@ -22,11 +23,15 @@ use tinymist_project::vfs::WorkspaceResolver;
use tinymist_project::{EntryReader, LspWorld, TaskInputs}; use tinymist_project::{EntryReader, LspWorld, TaskInputs};
use tinymist_std::error::prelude::*; use tinymist_std::error::prelude::*;
use typst::World; use typst::World;
use typst::WorldExt;
use typst::diag::SourceDiagnostic;
use typst::foundations::Bytes; use typst::foundations::Bytes;
use typst::html::HtmlDocument; use typst::html::HtmlDocument;
use typst_syntax::Span;
use typst_syntax::VirtualPath; use typst_syntax::VirtualPath;
pub use crate::common::Format; pub use crate::common::Format;
use crate::diagnostics::WarningCollector;
use crate::parser::HtmlToAstParser; use crate::parser::HtmlToAstParser;
use crate::writer::WriterFactory; use crate::writer::WriterFactory;
use typst_syntax::FileId; use typst_syntax::FileId;
@ -47,6 +52,7 @@ pub struct MarkdownDocument {
world: Arc<LspWorld>, world: Arc<LspWorld>,
feat: TypliteFeat, feat: TypliteFeat,
ast: Option<Node>, ast: Option<Node>,
warnings: WarningCollector,
} }
impl MarkdownDocument { impl MarkdownDocument {
@ -57,6 +63,7 @@ impl MarkdownDocument {
world, world,
feat, feat,
ast: None, ast: None,
warnings: WarningCollector::default(),
} }
} }
@ -72,15 +79,69 @@ impl MarkdownDocument {
world, world,
feat, feat,
ast: Some(ast), ast: Some(ast),
warnings: WarningCollector::default(),
} }
} }
/// Replace the backing warning collector, preserving shared state with
/// other components of the pipeline.
pub(crate) fn with_warning_collector(mut self, collector: WarningCollector) -> Self {
self.warnings = collector;
self
}
/// Get a snapshot of all collected warnings so far.
pub fn warnings(&self) -> Vec<SourceDiagnostic> {
let warnings = self.warnings.snapshot();
if let Some(info) = &self.feat.wrap_info {
warnings
.into_iter()
.filter_map(|diag| self.remap_diagnostic(diag, info))
.collect()
} else {
warnings
}
}
/// Internal accessor for sharing the collector with the parser.
fn warning_collector(&self) -> WarningCollector {
self.warnings.clone()
}
fn remap_diagnostic(
&self,
mut diagnostic: SourceDiagnostic,
info: &WrapInfo,
) -> Option<SourceDiagnostic> {
if let Some(span) = info.remap_span(self.world.as_ref(), diagnostic.span) {
diagnostic.span = span;
} else {
return None;
}
diagnostic.trace = diagnostic
.trace
.into_iter()
.filter_map(
|mut spanned| match info.remap_span(self.world.as_ref(), spanned.span) {
Some(span) => {
spanned.span = span;
Some(spanned)
}
None => None,
},
)
.collect();
Some(diagnostic)
}
/// Parse HTML document to AST /// Parse HTML document to AST
pub fn parse(&self) -> tinymist_std::Result<Node> { pub fn parse(&self) -> tinymist_std::Result<Node> {
if let Some(ast) = &self.ast { if let Some(ast) = &self.ast {
return Ok(ast.clone()); return Ok(ast.clone());
} }
let parser = HtmlToAstParser::new(self.feat.clone(), &self.world); let parser = HtmlToAstParser::new(self.feat.clone(), &self.world, self.warning_collector());
parser.parse(&self.base.root).context_ut("failed to parse") parser.parse(&self.base.root).context_ut("failed to parse")
} }
@ -141,6 +202,38 @@ pub enum ColorTheme {
Dark, Dark,
} }
#[derive(Debug, Clone)]
pub struct WrapInfo {
/// The synthetic wrapper file that hosts the original Typst source.
pub wrap_file_id: FileId,
/// The user's actual Typst source file.
pub original_file_id: FileId,
/// Number of UTF-8 bytes injected ahead of the original source.
pub prefix_len_bytes: usize,
}
impl WrapInfo {
/// Translate a span from the wrapper file back into the original file.
pub fn remap_span(&self, world: &dyn typst::World, span: Span) -> Option<Span> {
if span.id() != Some(self.wrap_file_id) {
return Some(span);
}
let range = world.range(span)?;
let start = range.start.checked_sub(self.prefix_len_bytes)?;
let end = range.end.checked_sub(self.prefix_len_bytes)?;
let original_source = world.source(self.original_file_id).ok()?;
let original_len = original_source.len_bytes();
if start >= original_len || end > original_len {
return None;
}
Some(Span::from_range(self.original_file_id, start..end))
}
}
#[derive(Debug, Default, Clone)] #[derive(Debug, Default, Clone)]
pub struct TypliteFeat { pub struct TypliteFeat {
/// The preferred color theme. /// The preferred color theme.
@ -178,6 +271,8 @@ pub struct TypliteFeat {
/// It resembles the regular typst show rule function, like `#show: /// It resembles the regular typst show rule function, like `#show:
/// article`. /// article`.
pub processor: Option<String>, pub processor: Option<String>,
/// Optional mapping from the wrapper file back to the original source.
pub wrap_info: Option<WrapInfo>,
} }
impl TypliteFeat { impl TypliteFeat {
@ -185,7 +280,7 @@ impl TypliteFeat {
&self, &self,
world: &LspWorld, world: &LspWorld,
format: Format, format: Format,
) -> tinymist_std::Result<LspWorld> { ) -> tinymist_std::Result<(LspWorld, Option<WrapInfo>)> {
let entry = world.entry_state(); let entry = world.entry_state();
let main = entry.main(); let main = entry.main();
let current = main.context("no main file in workspace")?; let current = main.context("no main file in workspace")?;
@ -242,19 +337,18 @@ impl TypliteFeat {
Bytes::from_string(include_str!("markdown.typ")), Bytes::from_string(include_str!("markdown.typ")),
) )
.context_ut("cannot map markdown.typ")?; .context_ut("cannot map markdown.typ")?;
let original_source = world
.source(current)
.context_ut("cannot fetch main source")?
.text()
.to_owned();
const WRAP_PREFIX: &str =
"#import \"@local/_markdown:0.1.0\": md-doc, example; #show: md-doc\n";
let wrap_content = format!("{WRAP_PREFIX}{original_source}");
world world
.map_shadow_by_id( .map_shadow_by_id(wrap_main_id, Bytes::from_string(wrap_content))
wrap_main_id,
Bytes::from_string(format!(
r#"#import "@local/_markdown:0.1.0": md-doc, example; #show: md-doc
{}"#,
world
.source(current)
.context_ut("failed to get main file content")?
.text()
)),
)
.context_ut("cannot map source for main file")?; .context_ut("cannot map source for main file")?;
if let Some(main_content) = main_content { if let Some(main_content) = main_content {
@ -263,7 +357,13 @@ impl TypliteFeat {
.context_ut("cannot map source for main file")?; .context_ut("cannot map source for main file")?;
} }
Ok(world) let wrap_info = Some(WrapInfo {
wrap_file_id: wrap_main_id,
original_file_id: current,
prefix_len_bytes: WRAP_PREFIX.len(),
});
Ok((world, wrap_info))
} }
} }
@ -319,9 +419,11 @@ impl Typlite {
} }
/// Convert the content to a markdown document. /// Convert the content to a markdown document.
pub fn convert_doc(self, format: Format) -> tinymist_std::Result<MarkdownDocument> { pub fn convert_doc(mut self, format: Format) -> tinymist_std::Result<MarkdownDocument> {
let world = Arc::new(self.feat.prepare_world(&self.world, format)?); let (prepared_world, wrap_info) = self.feat.prepare_world(&self.world, format)?;
self.feat.wrap_info = wrap_info;
let feat = self.feat.clone(); let feat = self.feat.clone();
let world = Arc::new(prepared_world);
Self::convert_doc_prepared(feat, format, world) Self::convert_doc_prepared(feat, format, world)
} }
@ -331,11 +433,22 @@ impl Typlite {
format: Format, format: Format,
world: Arc<LspWorld>, world: Arc<LspWorld>,
) -> tinymist_std::Result<MarkdownDocument> { ) -> tinymist_std::Result<MarkdownDocument> {
// todo: ignoring warnings let compiled = typst::compile(&world);
let base = typst::compile(&world).output?; let collector = WarningCollector::default();
collector.extend(
compiled
.warnings
.iter()
.filter(|&diag| {
diag.message.as_str()
!= "html export is under active development and incomplete"
})
.cloned(),
);
let base = compiled.output?;
let mut feat = feat; let mut feat = feat;
feat.target = format; feat.target = format;
Ok(MarkdownDocument::new(base, world.clone(), feat)) Ok(MarkdownDocument::new(base, world.clone(), feat).with_warning_collector(collector))
} }
} }

8
crates/typlite/src/main.rs
View file

@ -52,6 +52,7 @@ pub struct CompileArgs {
} }
fn main() -> Result<()> { fn main() -> Result<()> {
let _ = env_logger::try_init();
// Parse command line arguments // Parse command line arguments
let args = CompileArgs::parse(); let args = CompileArgs::parse();
@ -103,6 +104,8 @@ fn run(args: CompileArgs, world: Arc<LspWorld>) -> Result<()> {
Format::Docx => Bytes::new(doc.to_docx()?), Format::Docx => Bytes::new(doc.to_docx()?),
}; };
let warnings = doc.warnings();
if is_stdout { if is_stdout {
std::io::stdout() std::io::stdout()
.write_all(result.as_slice()) .write_all(result.as_slice())
@ -111,6 +114,11 @@ fn run(args: CompileArgs, world: Arc<LspWorld>) -> Result<()> {
bail!("failed to write file {output_path:?}: {err}"); bail!("failed to write file {output_path:?}: {err}");
} }
if !warnings.is_empty() {
print_diagnostics(world.as_ref(), warnings.iter(), DiagnosticFormat::Human)
.context_ut("print warnings")?;
}
Ok(()) Ok(())
} }

6
crates/typlite/src/markdown.typ
View file

@ -13,7 +13,7 @@
#let md-emph(body) = html.elem("span", html.elem("m1emph", body)) #let md-emph(body) = html.elem("span", html.elem("m1emph", body))
#let md-highlight(body) = html.elem("span", html.elem("m1highlight", body)) #let md-highlight(body) = html.elem("span", html.elem("m1highlight", body))
#let md-strike(body) = html.elem("span", html.elem("m1strike", body)) #let md-strike(body) = html.elem("span", html.elem("m1strike", body))
#let md-raw(lang: none, block: false, text) = { #let md-raw(lang: none, block: false, text: "", body) = {
let body = html.elem( let body = html.elem(
"m1raw", "m1raw",
attrs: ( attrs: (
@ -25,7 +25,7 @@
block: bool-str(block), block: bool-str(block),
text: text, text: text,
), ),
"", body,
) )
if block { if block {
@ -190,7 +190,7 @@
// todo: icc? // todo: icc?
show image: it => if-not-paged(it, md-image(src: it.source, alt: it.alt)) show image: it => if-not-paged(it, md-image(src: it.source, alt: it.alt))
show raw: it => if-not-paged(it, md-raw(lang: it.lang, block: it.block, it.text)) show raw: it => if-not-paged(it, md-raw(lang: it.lang, block: it.block, text: it.text, it))
show link: it => if-not-paged(it, md-link(dest: it.dest, it.body)) show link: it => if-not-paged(it, md-link(dest: it.dest, it.body))
show ref: it => if-not-paged(it, md-ref(it)) show ref: it => if-not-paged(it, md-ref(it))

35
crates/typlite/src/parser/core.rs
View file

@ -2,6 +2,9 @@
use std::sync::Arc; use std::sync::Arc;
use typst::diag::SourceDiagnostic;
use typst_syntax::Span;
use cmark_writer::WriteResult; use cmark_writer::WriteResult;
use cmark_writer::ast::{CustomNode, HtmlAttribute, HtmlElement as CmarkHtmlElement, Node}; use cmark_writer::ast::{CustomNode, HtmlAttribute, HtmlElement as CmarkHtmlElement, Node};
use cmark_writer::writer::InlineWriterProxy; use cmark_writer::writer::InlineWriterProxy;
@ -13,6 +16,7 @@ use crate::Result;
use crate::TypliteFeat; use crate::TypliteFeat;
use crate::attributes::{AlertsAttr, HeadingAttr, RawAttr, TypliteAttrsParser, md_attr}; use crate::attributes::{AlertsAttr, HeadingAttr, RawAttr, TypliteAttrsParser, md_attr};
use crate::common::{AlertNode, CenterNode, VerbatimNode}; use crate::common::{AlertNode, CenterNode, VerbatimNode};
use crate::diagnostics::WarningCollector;
use crate::tags::md_tag; use crate::tags::md_tag;
use super::{list::ListParser, table::TableParser}; use super::{list::ListParser, table::TableParser};
@ -25,10 +29,15 @@ pub struct HtmlToAstParser {
pub list_level: usize, pub list_level: usize,
pub blocks: Vec<Node>, pub blocks: Vec<Node>,
pub inline_buffer: Vec<Node>, pub inline_buffer: Vec<Node>,
pub(crate) warnings: WarningCollector,
} }
impl HtmlToAstParser { impl HtmlToAstParser {
pub fn new(feat: TypliteFeat, world: &Arc<LspWorld>) -> Self { pub(crate) fn new(
feat: TypliteFeat,
world: &Arc<LspWorld>,
warnings: WarningCollector,
) -> Self {
Self { Self {
feat, feat,
world: world.clone(), world: world.clone(),
@ -36,6 +45,7 @@ impl HtmlToAstParser {
list_level: 0, list_level: 0,
blocks: Vec::new(), blocks: Vec::new(),
inline_buffer: Vec::new(), inline_buffer: Vec::new(),
warnings,
} }
} }
@ -197,6 +207,12 @@ impl HtmlToAstParser {
let tag_name = element.tag.resolve().to_string(); let tag_name = element.tag.resolve().to_string();
if !tag_name.starts_with("m1") { if !tag_name.starts_with("m1") {
// self.warn_at(
// Some(element.span),
// eco_format!(
// "unsupported HTML element `<{tag_name}>`; exported as raw HTML"
// ),
// );
let html_element = self.create_html_element(element)?; let html_element = self.create_html_element(element)?;
self.inline_buffer.push(html_element); self.inline_buffer.push(html_element);
} else { } else {
@ -290,6 +306,23 @@ impl HtmlToAstParser {
Ok((inline, blocks)) Ok((inline, blocks))
} }
pub(crate) fn warn_at(&mut self, span: Option<Span>, message: EcoString) {
let span = span.unwrap_or_else(Span::detached);
let span = self
.feat
.wrap_info
.as_ref()
.and_then(|info| self.remap_span_from_wrapper(span, info))
.unwrap_or(span);
let diag = SourceDiagnostic::warning(span, message);
self.warnings.extend(std::iter::once(diag));
}
fn remap_span_from_wrapper(&self, span: Span, info: &crate::WrapInfo) -> Option<Span> {
info.remap_span(self.world.as_ref(), span)
}
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]

11
crates/typlite/src/parser/media.rs
View file

@ -7,6 +7,7 @@ use std::sync::{Arc, LazyLock};
use base64::Engine; use base64::Engine;
use cmark_writer::ast::{HtmlAttribute, HtmlElement as CmarkHtmlElement, Node}; use cmark_writer::ast::{HtmlAttribute, HtmlElement as CmarkHtmlElement, Node};
use ecow::{EcoString, eco_format}; use ecow::{EcoString, eco_format};
use log::debug;
use tinymist_project::diag::print_diagnostics_to_string; use tinymist_project::diag::print_diagnostics_to_string;
use tinymist_project::{EntryReader, MEMORY_MAIN_ENTRY, TaskInputs, base::ShadowApi}; use tinymist_project::{EntryReader, MEMORY_MAIN_ENTRY, TaskInputs, base::ShadowApi};
use typst::{ use typst::{
@ -210,7 +211,7 @@ impl HtmlToAstParser {
}); });
if self.feat.remove_html { if self.feat.remove_html {
eprintln!("Removing idoc element due to remove_html feature"); debug!("remove_html feature active, dropping inline document element");
// todo: make error silent is not good. // todo: make error silent is not good.
return Node::Text(EcoString::new()); return Node::Text(EcoString::new());
} }
@ -278,12 +279,12 @@ impl HtmlToAstParser {
) )
.unwrap(); .unwrap();
//todo: ignoring warnings let compiled = typst::compile(&world);
let doc = typst::compile(&world); self.warnings.extend(compiled.warnings.iter().cloned());
let doc = match doc.output { let doc = match compiled.output {
Ok(doc) => doc, Ok(doc) => doc,
Err(e) => { Err(e) => {
let diag = doc.warnings.iter().chain(e.iter()); let diag = compiled.warnings.iter().chain(e.iter());
let e = print_diagnostics_to_string( let e = print_diagnostics_to_string(
&world, &world,

19
crates/typlite/src/parser/table.rs
View file

@ -30,6 +30,12 @@ impl TableParser {
// Check if the table contains rowspan or colspan attributes // Check if the table contains rowspan or colspan attributes
// If it does, fall back to using HtmlElement // If it does, fall back to using HtmlElement
if Self::table_has_complex_cells(table) { if Self::table_has_complex_cells(table) {
parser.warn_at(
Some(table.span),
eco_format!(
"table contains rowspan or colspan attributes; exported original HTML table"
),
);
return parser.create_html_element(table).map(Some); return parser.create_html_element(table).map(Some);
} }
@ -48,15 +54,8 @@ impl TableParser {
)?; )?;
if fallback_to_html { if fallback_to_html {
eprintln!(
"[typlite] warning: block content detected inside table cell; exporting original HTML table"
);
let html = let html =
Self::serialize_html_element(parser, table).map_err(|e| e.to_string())?; Self::serialize_html_element(parser, table).map_err(|e| e.to_string())?;
let html = eco_format!(
"<!-- typlite warning: block content detected inside table cell; exported original HTML table -->\n{}",
html
);
return Ok(Some(Node::HtmlBlock(html))); return Ok(Some(Node::HtmlBlock(html)));
} }
@ -226,6 +225,12 @@ impl TableParser {
let (cell_content, block_content) = parser.capture_children(cell)?; let (cell_content, block_content) = parser.capture_children(cell)?;
if !block_content.is_empty() { if !block_content.is_empty() {
parser.warn_at(
Some(cell.span),
eco_format!(
"block content detected inside table cell; exported original HTML table"
),
);
*fallback_to_html = true; *fallback_to_html = true;
return Ok(Vec::new()); return Ok(Vec::new());
} }

8
crates/typlite/src/writer/docx/writer.rs
View file

@ -4,6 +4,7 @@ use base64::Engine;
use cmark_writer::ast::{ListItem, Node}; use cmark_writer::ast::{ListItem, Node};
use docx_rs::*; use docx_rs::*;
use ecow::EcoString; use ecow::EcoString;
use log::{debug, warn};
use std::fs; use std::fs;
use std::io::Cursor; use std::io::Cursor;
@ -252,11 +253,14 @@ impl DocxWriter {
} }
node if node.is_custom_type::<VerbatimNode>() => { node if node.is_custom_type::<VerbatimNode>() => {
let node = node.as_custom_type::<VerbatimNode>().unwrap(); let node = node.as_custom_type::<VerbatimNode>().unwrap();
eprintln!("Warning: `m1verbatim` is ignored {:?}.", node.content); warn!(
"ignoring `m1verbatim` content in DOCX export: {:?}",
node.content
);
} }
// Other inline element types // Other inline element types
_ => { _ => {
eprintln!("other inline element: {node:?}"); debug!("unhandled inline node in DOCX export: {node:?}");
} }
} }