language-servers/tinymist
1
0
Fork 0
mirror of https://github.com/Myriad-Dreamin/tinymist.git synced 2025-08-03 17:58:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 12

fix: ignore invalid tokens in typst's syntax tree (#605)

Browse source
This commit is contained in:
Myriad-Dreamin 2024-09-20 13:59:15 +08:00 • committed by GitHub
parent fcde0efb22
commit dc6a4482c1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 30 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/tinymist-query/src/fixtures/semantic_tokens/snaps/test@content-block.typ.snap
View file

@ -3,4 +3,4 @@ source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/content-block.typ
---
{"data":[0,0,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,20,0,1,0,0,20,0,0,0,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,1,20,0,1,0,0,20,0,0,0,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,3,20,0,0,3,1,8,0,0,1,1,8,0,0,1,1,8,0]}
{"data":[0,0,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,20,0,1,0,0,20,0,0,0,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,1,20,0,1,0,0,20,0,0,0,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,20,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,3,20,0,0,3,1,8,0,0,1,1,8,0,0,1,1,8,0,0,1,1,20,0,1,0,0,20,0]}

6
crates/tinymist-query/src/fixtures/semantic_tokens/snaps/test@tinymist_issue_601.typ.snap Normal file
View file

@ -0,0 +1,6 @@
---
source: crates/tinymist-query/src/semantic_tokens_full.rs
expression: "serde_json::to_string(&result).unwrap()"
input_file: crates/tinymist-query/src/fixtures/semantic_tokens/tinymist_issue_601.typ
---
{"data":[0,0,1,19,4,0,1,3,5,4,0,3,1,8,4,0,1,1,20,4,0,1,1,3,4,0,1,1,20,4,0,1,1,20,4,1,0,0,20,4,0,0,1,20,4,0,1,1,20,4,1,0,1,20,4,1,0,0,20,4,0,0,0,20,0]}

3
crates/tinymist-query/src/fixtures/semantic_tokens/tinymist_issue_601.typ Normal file
View file

@ -0,0 +1,3 @@
$abs(M_)
x

27
crates/tinymist-query/src/semantic_tokens/mod.rs
View file

@ -146,14 +146,25 @@ impl Tokenizer {
}
fn push(&mut self, token: Token) {
let Token {
token_type,
modifiers,
range,
} = token;
use crate::typst_to_lsp;
use lsp_types::Position;
let utf8_start = token.range.start;
let utf8_start = range.start;
if self.pos_offset > utf8_start {
return;
}
let utf8_end = token.range.end;
// This might be a bug of typst, that `end > len` is possible
let utf8_end = (range.end).min(self.source.text().len());
self.pos_offset = utf8_start;
if utf8_end < range.start || range.start > self.source.text().len() {
return;
}
let position = typst_to_lsp::offset_to_position(utf8_start, self.encoding, &self.source);
@ -177,8 +188,8 @@ impl Tokenizer {
delta_line: delta.delta_line,
delta_start: delta.delta_start,
length: encode_length(utf8_start, utf8_end) as u32,
token_type: token.token_type as u32,
token_modifiers_bitset: token.modifiers.bitset(),
token_type: token_type as u32,
token_modifiers_bitset: modifiers.bitset(),
});
} else {
let final_line = self
@ -193,8 +204,8 @@ impl Tokenizer {
delta_line: delta.delta_line,
delta_start: delta.delta_start,
length: encode_length(utf8_start, utf8_end.min(next_offset)) as u32,
token_type: token.token_type as u32,
token_modifiers_bitset: token.modifiers.bitset(),
token_type: token_type as u32,
token_modifiers_bitset: modifiers.bitset(),
});
let mut utf8_cursor = next_offset;
if self.curr_pos.line < final_line {
@ -211,8 +222,8 @@ impl Tokenizer {
delta_line: 1,
delta_start: 0,
length: encode_length(utf8_cursor, next_offset) as u32,
token_type: token.token_type as u32,
token_modifiers_bitset: token.modifiers.bitset(),
token_type: token_type as u32,
token_modifiers_bitset: modifiers.bitset(),
});
self.pos_offset = utf8_cursor;
utf8_cursor = next_offset;

2
crates/tinymist-query/src/tests.rs
View file

@ -148,7 +148,7 @@ pub fn run_with_sources<T>(source: &str, f: impl FnOnce(&mut LspUniverse, PathBu
let mut last_pw = None;
for (i, source) in sources.enumerate() {
// find prelude
let mut source = source.trim();
let mut source = source.trim_start();
let mut path = None;
if source.starts_with("//") {