From de1f766820d58bd87a94e9f055fbf269a3946e1f Mon Sep 17 00:00:00 2001 From: oxalica Date: Mon, 17 Jul 2023 22:28:57 +0800 Subject: [PATCH 1/4] Fix highlighting of byte escape sequences Currently non-UTF8 escape sequences in byte strings and any escape sequences in byte literals are ignored. --- crates/ide/src/syntax_highlighting.rs | 10 ++++++++- crates/ide/src/syntax_highlighting/escape.rs | 22 ++++++++++++++++++- .../test_data/highlight_strings.html | 6 +++-- crates/ide/src/syntax_highlighting/tests.rs | 6 +++-- crates/syntax/src/ast/token_ext.rs | 14 ++++++++---- 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs index 577bd2bc1f..ae97236409 100644 --- a/crates/ide/src/syntax_highlighting.rs +++ b/crates/ide/src/syntax_highlighting.rs @@ -24,7 +24,7 @@ use syntax::{ use crate::{ syntax_highlighting::{ - escape::{highlight_escape_char, highlight_escape_string}, + escape::{highlight_escape_byte, highlight_escape_char, highlight_escape_string}, format::highlight_format_string, highlights::Highlights, macro_::MacroHighlighter, @@ -471,6 +471,14 @@ fn traverse( }; highlight_escape_char(hl, &char, range.start()) + } else if ast::Byte::can_cast(token.kind()) + && ast::Byte::can_cast(descended_token.kind()) + { + let Some(byte) = ast::Byte::cast(token) else { + continue; + }; + + highlight_escape_byte(hl, &byte, range.start()) } } diff --git a/crates/ide/src/syntax_highlighting/escape.rs b/crates/ide/src/syntax_highlighting/escape.rs index 211e358809..2c63a69bdc 100644 --- a/crates/ide/src/syntax_highlighting/escape.rs +++ b/crates/ide/src/syntax_highlighting/escape.rs @@ -1,7 +1,7 @@ //! Syntax highlighting for escape sequences use crate::syntax_highlighting::highlights::Highlights; use crate::{HlRange, HlTag}; -use syntax::ast::{Char, IsString}; +use syntax::ast::{Byte, Char, IsString}; use syntax::{AstToken, TextRange, TextSize}; pub(super) fn highlight_escape_string( @@ -43,3 +43,23 @@ pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char, start: TextRange::new(start + TextSize::from(1), start + TextSize::from(text.len() as u32 + 1)); stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None }) } + +pub(super) fn highlight_escape_byte(stack: &mut Highlights, byte: &Byte, start: TextSize) { + if byte.value().is_none() { + return; + } + + let text = byte.text(); + if !text.starts_with("b'") || !text.ends_with('\'') { + return; + } + + let text = &text[2..text.len() - 1]; + if !text.starts_with('\\') { + return; + } + + let range = + TextRange::new(start + TextSize::from(2), start + TextSize::from(text.len() as u32 + 2)); + stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None }) +} diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index f4f164aa1d..061329d239 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -105,6 +105,8 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd let a = '\x65'; let a = '\x00'; + let a = b'\xFF'; + println!("Hello {{Hello}}"); // from https://doc.rust-lang.org/std/fmt/index.html println!("Hello"); // => "Hello" @@ -159,8 +161,8 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd println!("Hello\nWorld"); println!("\u{48}\x65\x6C\x6C\x6F World"); - let _ = "\x28\x28\x00\x63\n"; - let _ = b"\x28\x28\x00\x63\n"; + let _ = "\x28\x28\x00\x63\xFF\n"; // invalid non-UTF8 escape sequences + let _ = b"\x28\x28\x00\x63\xFF\n"; // valid bytes let backslash = r"\\"; println!("{\x41}", A = 92); diff --git a/crates/ide/src/syntax_highlighting/tests.rs b/crates/ide/src/syntax_highlighting/tests.rs index 1ee451a06d..80a49bcaa3 100644 --- a/crates/ide/src/syntax_highlighting/tests.rs +++ b/crates/ide/src/syntax_highlighting/tests.rs @@ -451,6 +451,8 @@ fn main() { let a = '\x65'; let a = '\x00'; + let a = b'\xFF'; + println!("Hello {{Hello}}"); // from https://doc.rust-lang.org/std/fmt/index.html println!("Hello"); // => "Hello" @@ -505,8 +507,8 @@ fn main() { println!("Hello\nWorld"); println!("\u{48}\x65\x6C\x6C\x6F World"); - let _ = "\x28\x28\x00\x63\n"; - let _ = b"\x28\x28\x00\x63\n"; + let _ = "\x28\x28\x00\x63\xFF\n"; // invalid non-UTF8 escape sequences + let _ = b"\x28\x28\x00\x63\xFF\n"; // valid bytes let backslash = r"\\"; println!("{\x41}", A = 92); diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 090eb89f47..aa8c9bbc0f 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -146,6 +146,7 @@ impl QuoteOffsets { pub trait IsString: AstToken { const RAW_PREFIX: &'static str; + const MODE: Mode; fn is_raw(&self) -> bool { self.text().starts_with(Self::RAW_PREFIX) } @@ -181,7 +182,7 @@ pub trait IsString: AstToken { let text = &self.text()[text_range_no_quotes - start]; let offset = text_range_no_quotes.start() - start; - unescape_literal(text, Mode::Str, &mut |range, unescaped_char| { + unescape_literal(text, Self::MODE, &mut |range, unescaped_char| { let text_range = TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()); cb(text_range + offset, unescaped_char); @@ -196,6 +197,7 @@ pub trait IsString: AstToken { impl IsString for ast::String { const RAW_PREFIX: &'static str = "r"; + const MODE: Mode = Mode::Str; } impl ast::String { @@ -213,7 +215,7 @@ impl ast::String { let mut buf = String::new(); let mut prev_end = 0; let mut has_error = false; - unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match ( + unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match ( unescaped_char, buf.capacity() == 0, ) { @@ -239,6 +241,7 @@ impl ast::String { impl IsString for ast::ByteString { const RAW_PREFIX: &'static str = "br"; + const MODE: Mode = Mode::ByteStr; } impl ast::ByteString { @@ -256,7 +259,7 @@ impl ast::ByteString { let mut buf: Vec = Vec::new(); let mut prev_end = 0; let mut has_error = false; - unescape_literal(text, Mode::ByteStr, &mut |char_range, unescaped_char| match ( + unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match ( unescaped_char, buf.capacity() == 0, ) { @@ -282,6 +285,9 @@ impl ast::ByteString { impl IsString for ast::CString { const RAW_PREFIX: &'static str = "cr"; + // XXX: `Mode::CStr` is not supported by `unescape_literal` of ra-ap-rustc_lexer yet. + // Here we pretend it to be a byte string. + const MODE: Mode = Mode::ByteStr; } impl ast::CString { @@ -299,7 +305,7 @@ impl ast::CString { let mut buf = String::new(); let mut prev_end = 0; let mut has_error = false; - unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match ( + unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match ( unescaped_char, buf.capacity() == 0, ) { From 59a3e42ac9d6ad36b4af3732b46852a61647540c Mon Sep 17 00:00:00 2001 From: oxalica Date: Tue, 18 Jul 2023 18:42:02 +0800 Subject: [PATCH 2/4] Fix unescaping of C string literals --- crates/hir-def/src/body/pretty.rs | 2 +- crates/hir-def/src/hir.rs | 2 +- crates/hir-ty/src/mir/lower.rs | 1 - .../test_data/highlight_strings.html | 5 +- crates/ide/src/syntax_highlighting/tests.rs | 5 +- crates/syntax/src/ast/token_ext.rs | 55 ++++++++++++++----- 6 files changed, 49 insertions(+), 21 deletions(-) diff --git a/crates/hir-def/src/body/pretty.rs b/crates/hir-def/src/body/pretty.rs index 0c6cf0b49a..eeaed87164 100644 --- a/crates/hir-def/src/body/pretty.rs +++ b/crates/hir-def/src/body/pretty.rs @@ -634,7 +634,7 @@ impl Printer<'_> { match literal { Literal::String(it) => w!(self, "{:?}", it), Literal::ByteString(it) => w!(self, "\"{}\"", it.escape_ascii()), - Literal::CString(it) => w!(self, "\"{}\\0\"", it), + Literal::CString(it) => w!(self, "\"{}\\0\"", it.escape_ascii()), Literal::Char(it) => w!(self, "'{}'", it.escape_debug()), Literal::Bool(it) => w!(self, "{}", it), Literal::Int(i, suffix) => { diff --git a/crates/hir-def/src/hir.rs b/crates/hir-def/src/hir.rs index 500e880061..8a140a1ec1 100644 --- a/crates/hir-def/src/hir.rs +++ b/crates/hir-def/src/hir.rs @@ -85,7 +85,7 @@ impl fmt::Display for FloatTypeWrapper { pub enum Literal { String(Box), ByteString(Box<[u8]>), - CString(Box), + CString(Box<[u8]>), Char(char), Bool(bool), Int(i128, Option), diff --git a/crates/hir-ty/src/mir/lower.rs b/crates/hir-ty/src/mir/lower.rs index 3610858790..8da12f9e39 100644 --- a/crates/hir-ty/src/mir/lower.rs +++ b/crates/hir-ty/src/mir/lower.rs @@ -1355,7 +1355,6 @@ impl<'ctx> MirLowerCtx<'ctx> { return Ok(Operand::from_concrete_const(data, mm, ty)); } hir_def::hir::Literal::CString(b) => { - let b = b.as_bytes(); let bytes = b.iter().copied().chain(iter::once(0)).collect::>(); let mut data = Vec::with_capacity(mem::size_of::() * 2); diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index 061329d239..33523e4af7 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -161,8 +161,9 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd println!("Hello\nWorld"); println!("\u{48}\x65\x6C\x6C\x6F World"); - let _ = "\x28\x28\x00\x63\xFF\n"; // invalid non-UTF8 escape sequences - let _ = b"\x28\x28\x00\x63\xFF\n"; // valid bytes + let _ = "\x28\x28\x00\x63\xFF\u{FF}\n"; // invalid non-UTF8 escape sequences + let _ = b"\x28\x28\x00\x63\xFF\u{FF}\n"; // valid bytes, invalid unicodes + let _ = c"\u{FF}\xFF"; // valid bytes, valid unicodes let backslash = r"\\"; println!("{\x41}", A = 92); diff --git a/crates/ide/src/syntax_highlighting/tests.rs b/crates/ide/src/syntax_highlighting/tests.rs index 80a49bcaa3..696aa59002 100644 --- a/crates/ide/src/syntax_highlighting/tests.rs +++ b/crates/ide/src/syntax_highlighting/tests.rs @@ -507,8 +507,9 @@ fn main() { println!("Hello\nWorld"); println!("\u{48}\x65\x6C\x6C\x6F World"); - let _ = "\x28\x28\x00\x63\xFF\n"; // invalid non-UTF8 escape sequences - let _ = b"\x28\x28\x00\x63\xFF\n"; // valid bytes + let _ = "\x28\x28\x00\x63\xFF\u{FF}\n"; // invalid non-UTF8 escape sequences + let _ = b"\x28\x28\x00\x63\xFF\u{FF}\n"; // valid bytes, invalid unicodes + let _ = c"\u{FF}\xFF"; // valid bytes, valid unicodes let backslash = r"\\"; println!("{\x41}", A = 92); diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index aa8c9bbc0f..87fd51d703 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -2,7 +2,9 @@ use std::borrow::Cow; -use rustc_lexer::unescape::{unescape_byte, unescape_char, unescape_literal, Mode}; +use rustc_lexer::unescape::{ + unescape_byte, unescape_c_string, unescape_char, unescape_literal, CStrUnit, Mode, +}; use crate::{ ast::{self, AstToken}, @@ -285,45 +287,70 @@ impl ast::ByteString { impl IsString for ast::CString { const RAW_PREFIX: &'static str = "cr"; - // XXX: `Mode::CStr` is not supported by `unescape_literal` of ra-ap-rustc_lexer yet. - // Here we pretend it to be a byte string. - const MODE: Mode = Mode::ByteStr; + const MODE: Mode = Mode::CStr; + + fn escaped_char_ranges( + &self, + cb: &mut dyn FnMut(TextRange, Result), + ) { + let text_range_no_quotes = match self.text_range_between_quotes() { + Some(it) => it, + None => return, + }; + + let start = self.syntax().text_range().start(); + let text = &self.text()[text_range_no_quotes - start]; + let offset = text_range_no_quotes.start() - start; + + unescape_c_string(text, Self::MODE, &mut |range, unescaped_char| { + let text_range = + TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()); + // XXX: This method should only be used for highlighting ranges. The unescaped + // char/byte is not used. For simplicity, we return an arbitrary placeholder char. + cb(text_range + offset, unescaped_char.map(|_| ' ')); + }); + } } impl ast::CString { - pub fn value(&self) -> Option> { + pub fn value(&self) -> Option> { if self.is_raw() { let text = self.text(); let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; - return Some(Cow::Borrowed(text)); + return Some(Cow::Borrowed(text.as_bytes())); } let text = self.text(); let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()]; - let mut buf = String::new(); + let mut buf = Vec::new(); let mut prev_end = 0; let mut has_error = false; - unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match ( - unescaped_char, + let mut char_buf = [0u8; 4]; + let mut extend_unit = |buf: &mut Vec, unit: CStrUnit| match unit { + CStrUnit::Byte(b) => buf.push(b), + CStrUnit::Char(c) => buf.extend(c.encode_utf8(&mut char_buf).as_bytes()), + }; + unescape_c_string(text, Self::MODE, &mut |char_range, unescaped| match ( + unescaped, buf.capacity() == 0, ) { - (Ok(c), false) => buf.push(c), + (Ok(u), false) => extend_unit(&mut buf, u), (Ok(_), true) if char_range.len() == 1 && char_range.start == prev_end => { prev_end = char_range.end } - (Ok(c), true) => { + (Ok(u), true) => { buf.reserve_exact(text.len()); - buf.push_str(&text[..prev_end]); - buf.push(c); + buf.extend(text[..prev_end].as_bytes()); + extend_unit(&mut buf, u); } (Err(_), _) => has_error = true, }); match (has_error, buf.capacity() == 0) { (true, _) => None, - (false, true) => Some(Cow::Borrowed(text)), + (false, true) => Some(Cow::Borrowed(text.as_bytes())), (false, false) => Some(Cow::Owned(buf)), } } From 1f35e4d3f1a741b4fd7468dd62eaac7b14f184a7 Mon Sep 17 00:00:00 2001 From: oxalica Date: Wed, 19 Jul 2023 15:12:53 +0800 Subject: [PATCH 3/4] Introduce `invalidEscapeSequence` semantic token type --- crates/ide/src/syntax_highlighting/escape.rs | 10 +++++----- crates/ide/src/syntax_highlighting/html.rs | 3 ++- crates/ide/src/syntax_highlighting/tags.rs | 2 ++ .../test_data/highlight_assoc_functions.html | 3 ++- .../test_data/highlight_attributes.html | 3 ++- .../test_data/highlight_crate_root.html | 3 ++- .../test_data/highlight_default_library.html | 3 ++- .../test_data/highlight_doctest.html | 3 ++- .../test_data/highlight_extern_crate.html | 3 ++- .../test_data/highlight_general.html | 3 ++- .../test_data/highlight_injection.html | 3 ++- .../test_data/highlight_keywords.html | 3 ++- .../test_data/highlight_lifetimes.html | 3 ++- .../test_data/highlight_macros.html | 3 ++- .../test_data/highlight_module_docs_inline.html | 3 ++- .../test_data/highlight_module_docs_outline.html | 3 ++- .../test_data/highlight_operators.html | 3 ++- .../test_data/highlight_rainbow.html | 3 ++- .../test_data/highlight_strings.html | 7 ++++--- .../test_data/highlight_unsafe.html | 3 ++- crates/rust-analyzer/src/semantic_tokens.rs | 1 + crates/rust-analyzer/src/to_proto.rs | 1 + editors/code/package.json | 6 +++++- 23 files changed, 52 insertions(+), 26 deletions(-) diff --git a/crates/ide/src/syntax_highlighting/escape.rs b/crates/ide/src/syntax_highlighting/escape.rs index 2c63a69bdc..41b27683f6 100644 --- a/crates/ide/src/syntax_highlighting/escape.rs +++ b/crates/ide/src/syntax_highlighting/escape.rs @@ -10,14 +10,14 @@ pub(super) fn highlight_escape_string( start: TextSize, ) { string.escaped_char_ranges(&mut |piece_range, char| { - if char.is_err() { - return; - } - if string.text()[piece_range.start().into()..].starts_with('\\') { + let highlight = match char { + Ok(_) => HlTag::EscapeSequence, + Err(_) => HlTag::InvalidEscapeSequence, + }; stack.add(HlRange { range: piece_range + start, - highlight: HlTag::EscapeSequence.into(), + highlight: highlight.into(), binding_hash: None, }); } diff --git a/crates/ide/src/syntax_highlighting/html.rs b/crates/ide/src/syntax_highlighting/html.rs index 2c7823069b..bbc6b55a64 100644 --- a/crates/ide/src/syntax_highlighting/html.rs +++ b/crates/ide/src/syntax_highlighting/html.rs @@ -109,6 +109,7 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd .control { font-style: italic; } .reference { font-style: italic; font-weight: bold; } -.unresolved_reference { color: #FC5555; text-decoration: wavy underline; } +.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; } +.unresolved_reference { color: #FC5555; text-decoration: wavy underline; } "; diff --git a/crates/ide/src/syntax_highlighting/tags.rs b/crates/ide/src/syntax_highlighting/tags.rs index f983109115..6d4cdd0efe 100644 --- a/crates/ide/src/syntax_highlighting/tags.rs +++ b/crates/ide/src/syntax_highlighting/tags.rs @@ -29,6 +29,7 @@ pub enum HlTag { Comment, EscapeSequence, FormatSpecifier, + InvalidEscapeSequence, Keyword, NumericLiteral, Operator(HlOperator), @@ -166,6 +167,7 @@ impl HlTag { HlTag::CharLiteral => "char_literal", HlTag::Comment => "comment", HlTag::EscapeSequence => "escape_sequence", + HlTag::InvalidEscapeSequence => "invalid_escape_sequence", HlTag::FormatSpecifier => "format_specifier", HlTag::Keyword => "keyword", HlTag::Punctuation(punct) => match punct { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html b/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html index 9ed65fbc85..4dcbfe4eb6 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html @@ -40,7 +40,8 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd .control { font-style: italic; } .reference { font-style: italic; font-weight: bold; } -.unresolved_reference { color: #FC5555; text-decoration: wavy underline; } +.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; } +.unresolved_reference { color: #FC5555; text-decoration: wavy underline; } 
fn not_static() {}
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html b/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
index 567ab8ccc1..bf5505caf3 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
#[allow(dead_code)]
 #[rustfmt::skip]
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html b/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
index 1e4c06df7e..0d1b3c1f18 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
extern crate foo;
 use core::iter;
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html b/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
index 5d66f832da..dd1528ed03 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
use core::iter;
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
index 35f240d428..d5f92aa5d4 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
//! This is a module to test doc injection.
 //! ```
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html b/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
index 87b9da46e2..b15f7bca72 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
extern crate std;
 extern crate alloc as abc;
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_general.html b/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
index 6b049f379a..bdeb09d2f8 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
use inner::{self as inner_mod};
 mod inner {}
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
index d9c3db6fbb..f9c33b8a60 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
fn fixture(ra_fixture: &str) {}
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html b/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
index 3900959bed..fd3b39855e 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
extern crate self;
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html b/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
index f98e0b1cda..ec39998de2 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 

 #[derive()]
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html b/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
index 2cbbf69641..c5fcec7568 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
proc_macros::mirror! {
     {
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
index 8a1d69816e..4dcf8e5f01 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
//! [Struct]
 //! This is an intra doc injection test for modules
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
index c4c3e3dc26..084bbf2f74 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
/// [crate::foo::Struct]
 /// This is an intra doc injection test for modules
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html b/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
index 2369071ae2..1af4bcfbd9 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
fn main() {
     1 + 1 - 1 * 1 / 1 % 1 | 1 & 1 ! 1 ^ 1 >> 1 << 1;
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html b/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
index bff35c897e..ec18c3ea1f 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
fn main() {
     let hello = "hello";
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
index 33523e4af7..dcac8eb736 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
macro_rules! println {
     ($($arg:tt)*) => ({
@@ -161,8 +162,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
     println!("Hello\nWorld");
     println!("\u{48}\x65\x6C\x6C\x6F World");
 
-    let _ = "\x28\x28\x00\x63\xFF\u{FF}\n"; // invalid non-UTF8 escape sequences
-    let _ = b"\x28\x28\x00\x63\xFF\u{FF}\n"; // valid bytes, invalid unicodes
+    let _ = "\x28\x28\x00\x63\xFF\u{FF}\n"; // invalid non-UTF8 escape sequences
+    let _ = b"\x28\x28\x00\x63\xFF\u{FF}\n"; // valid bytes, invalid unicodes
     let _ = c"\u{FF}\xFF"; // valid bytes, valid unicodes
     let backslash = r"\\";
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html b/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
index 654d51b8a4..c72ea54e94 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
@@ -40,7 +40,8 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 
 
macro_rules! id {
     ($($tt:tt)*) => {
diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs
index d4bb20c8f4..1fe02fc7ea 100644
--- a/crates/rust-analyzer/src/semantic_tokens.rs
+++ b/crates/rust-analyzer/src/semantic_tokens.rs
@@ -78,6 +78,7 @@ define_semantic_token_types![
         (DERIVE_HELPER, "deriveHelper") => DECORATOR,
         (DOT, "dot"),
         (ESCAPE_SEQUENCE, "escapeSequence") => STRING,
+        (INVALID_ESCAPE_SEQUENCE, "invalidEscapeSequence") => STRING,
         (FORMAT_SPECIFIER, "formatSpecifier") => STRING,
         (GENERIC, "generic") => TYPE_PARAMETER,
         (LABEL, "label"),
diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
index ba3421bf9e..7a89533a5e 100644
--- a/crates/rust-analyzer/src/to_proto.rs
+++ b/crates/rust-analyzer/src/to_proto.rs
@@ -640,6 +640,7 @@ fn semantic_token_type_and_modifiers(
         HlTag::CharLiteral => semantic_tokens::CHAR,
         HlTag::Comment => semantic_tokens::COMMENT,
         HlTag::EscapeSequence => semantic_tokens::ESCAPE_SEQUENCE,
+        HlTag::InvalidEscapeSequence => semantic_tokens::INVALID_ESCAPE_SEQUENCE,
         HlTag::FormatSpecifier => semantic_tokens::FORMAT_SPECIFIER,
         HlTag::Keyword => semantic_tokens::KEYWORD,
         HlTag::None => semantic_tokens::GENERIC,
diff --git a/editors/code/package.json b/editors/code/package.json
index ffb5dd9079..c84fc1cd8a 100644
--- a/editors/code/package.json
+++ b/editors/code/package.json
@@ -1801,12 +1801,16 @@
             },
             {
                 "id": "escapeSequence",
-                "description": "Style for char escapes in strings"
+                "description": "Style for char or byte escapes in strings"
             },
             {
                 "id": "formatSpecifier",
                 "description": "Style for {} placeholders in format strings"
             },
+            {
+                "id": "invalidEscapeSequence",
+                "description": "Style for invalid char or byte escapes in strings"
+            },
             {
                 "id": "label",
                 "description": "Style for labels"

From 51b35ccb1b134d511cd29ea4a06c8d1abaaffabb Mon Sep 17 00:00:00 2001
From: oxalica 
Date: Sun, 23 Jul 2023 04:24:35 +0800
Subject: [PATCH 4/4] Add comments for why skip highlighting for invalid
 char/byte literals

---
 crates/ide/src/syntax_highlighting/escape.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/crates/ide/src/syntax_highlighting/escape.rs b/crates/ide/src/syntax_highlighting/escape.rs
index 41b27683f6..5913ca5e45 100644
--- a/crates/ide/src/syntax_highlighting/escape.rs
+++ b/crates/ide/src/syntax_highlighting/escape.rs
@@ -26,6 +26,9 @@ pub(super) fn highlight_escape_string(
 
 pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char, start: TextSize) {
     if char.value().is_none() {
+        // We do not emit invalid escapes highlighting here. The lexer would likely be in a bad
+        // state and this token contains junks, since `'` is not a reliable delimiter (consider
+        // lifetimes). Nonetheless, parser errors should already be emitted.
         return;
     }
 
@@ -46,6 +49,7 @@ pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char, start:
 
 pub(super) fn highlight_escape_byte(stack: &mut Highlights, byte: &Byte, start: TextSize) {
     if byte.value().is_none() {
+        // See `highlight_escape_char` for why no error highlighting here.
         return;
     }