fix: set position_encoding on handshake and fix to utf-16 (#2153)

Although we don't have enough tests about utf-8 position encoding, i.e.
it is not good to decide to use utf-8, it is completely wrong to not
passing a decided encoding back to client on initialization.

Edit: it doesn't work correctly under utf-8, so we currently always use
utf-16. We may fix it in v0.14

Dockerfile

@@ -8,7 +8,7 @@
 # https://stackoverflow.com/a/64528456
 # https://depot.dev/blog/rust-dockerfile-best-practices
 
-ARG RUST_VERSION=1.88.0
+ARG RUST_VERSION=1.89.0
 
 FROM rust:${RUST_VERSION}-bookworm AS base
 RUN apt-get install -y git
@@ -40,7 +40,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
 COPY . .
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=$SCCACHE_DIR,sharing=locked \
-    cargo +${RUST_VERSION} build -p tinymist --release
+    cargo +${RUST_VERSION} build --bin tinymist --release
 
 FROM debian:12
 WORKDIR /app/

crates/tinymist/src/config.rs

@@ -769,18 +769,22 @@ impl Default for ConstConfig {
 
 impl From<&InitializeParams> for ConstConfig {
     fn from(params: &InitializeParams) -> Self {
-        const DEFAULT_ENCODING: &[PositionEncodingKind] = &[PositionEncodingKind::UTF16];
+        // const DEFAULT_ENCODING: &[PositionEncodingKind] =
+        // &[PositionEncodingKind::UTF16];
 
+        // todo: respect position encoding.
         let position_encoding = {
-            let general = params.capabilities.general.as_ref();
+            // let general = params.capabilities.general.as_ref();
-            let encodings = try_(|| Some(general?.position_encodings.as_ref()?.as_slice()));
+            // let encodings = try_(||
-            let encodings = encodings.unwrap_or(DEFAULT_ENCODING);
+            // Some(general?.position_encodings.as_ref()?.as_slice()));
+            // let encodings = encodings.unwrap_or(DEFAULT_ENCODING);
 
-            if encodings.contains(&PositionEncodingKind::UTF8) {
+            // if encodings.contains(&PositionEncodingKind::UTF8) {
-                PositionEncoding::Utf8
+            //     PositionEncoding::Utf8
-            } else {
+            // } else {
+            //     PositionEncoding::Utf16
+            // }
             PositionEncoding::Utf16
-            }
         };
 
         let workspace = params.capabilities.workspace.as_ref();

crates/tinymist/src/lsp/init.rs

@@ -123,8 +123,7 @@ impl Initializer for SuperInit {
 
         let res = InitializeResult {
             capabilities: ServerCapabilities {
-                // todo: respect position_encoding
+                position_encoding: Some(const_config.position_encoding.into()),
-                // position_encoding: Some(cc.position_encoding.into()),
                 hover_provider: Some(HoverProviderCapability::Simple(true)),
                 signature_help_provider: Some(SignatureHelpOptions {
                     trigger_characters: Some(vec![

editors/neovim/samples/lazyvim-dev/Dockerfile

@@ -24,7 +24,7 @@ COPY --from=builder /neovim/build/nvim-linux-x86_64.deb /tmp/nvim-linux-x86_64.d
 RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
     --mount=target=/var/cache/apt,type=cache,sharing=locked \
     rm -f /etc/apt/apt.conf.d/docker-clean && \
-    apt-get update && apt-get install -y curl git ripgrep build-essential unzip
+    apt-get update && apt-get install -y curl wget git ripgrep build-essential unzip
 RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \
     --mount=target=/var/cache/apt,type=cache,sharing=locked \
     rm -f /etc/apt/apt.conf.d/docker-clean && \

editors/neovim/samples/lazyvim-dev/plugins/dev.lua

@@ -3,6 +3,8 @@ local tinymist = require("others.tinymist")[2]
 -- set binary path
 tinymist.opts.servers.tinymist.cmd = { "tinymist" }
 
+vim.lsp.config("tinymist", tinymist.opts.servers.tinymist)
+
 
 return {
     { "mason-org/mason.nvim",           version = "^1.0.0" },

tests/e2e/e2e/lsp.rs

@@ -22,7 +22,7 @@ fn test_lsp() {
         });
 
         let hash = replay_log(&root.join("neovim"));
-        insta::assert_snapshot!(hash, @"siphash128_13:6748f1ff6d06426aa788a0683b6146d5");
+        insta::assert_snapshot!(hash, @"siphash128_13:320c08e4c9458c7bf2c8ce0f2aecf05d");
     }
 
     {
@@ -33,7 +33,7 @@ fn test_lsp() {
         });
 
         let hash = replay_log(&root.join("vscode"));
-        insta::assert_snapshot!(hash, @"siphash128_13:37838db38884e581e48751286df407d0");
+        insta::assert_snapshot!(hash, @"siphash128_13:f3723c32f48a55ecf1b86af639002af5");
     }
 }
 

tests/workspaces/individuals/utf8-encoding.typ

@@ -0,0 +1,2 @@
+Input was a single line with content:
+$𝒮(G)$ foo bar $bold(f)$.