Merge commit '99718d0c8b' into sync-from-ra

crates/syntax/src/ast/token_ext.rs

@@ -2,7 +2,9 @@
 
 use std::borrow::Cow;
 
-use rustc_lexer::unescape::{unescape_byte, unescape_char, unescape_literal, Mode};
+use rustc_lexer::unescape::{
+    unescape_byte, unescape_c_string, unescape_char, unescape_literal, CStrUnit, Mode,
+};
 
 use crate::{
     ast::{self, AstToken},
@@ -146,6 +148,7 @@ impl QuoteOffsets {
 
 pub trait IsString: AstToken {
     const RAW_PREFIX: &'static str;
+    const MODE: Mode;
     fn is_raw(&self) -> bool {
         self.text().starts_with(Self::RAW_PREFIX)
     }
@@ -181,7 +184,7 @@ pub trait IsString: AstToken {
         let text = &self.text()[text_range_no_quotes - start];
         let offset = text_range_no_quotes.start() - start;
 
-        unescape_literal(text, Mode::Str, &mut |range, unescaped_char| {
+        unescape_literal(text, Self::MODE, &mut |range, unescaped_char| {
             let text_range =
                 TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap());
             cb(text_range + offset, unescaped_char);
@@ -196,6 +199,7 @@ pub trait IsString: AstToken {
 
 impl IsString for ast::String {
     const RAW_PREFIX: &'static str = "r";
+    const MODE: Mode = Mode::Str;
 }
 
 impl ast::String {
@@ -213,7 +217,7 @@ impl ast::String {
         let mut buf = String::new();
         let mut prev_end = 0;
         let mut has_error = false;
-        unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match (
+        unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match (
             unescaped_char,
             buf.capacity() == 0,
         ) {
@@ -239,6 +243,7 @@ impl ast::String {
 
 impl IsString for ast::ByteString {
     const RAW_PREFIX: &'static str = "br";
+    const MODE: Mode = Mode::ByteStr;
 }
 
 impl ast::ByteString {
@@ -256,7 +261,7 @@ impl ast::ByteString {
         let mut buf: Vec<u8> = Vec::new();
         let mut prev_end = 0;
         let mut has_error = false;
-        unescape_literal(text, Mode::ByteStr, &mut |char_range, unescaped_char| match (
+        unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match (
             unescaped_char,
             buf.capacity() == 0,
         ) {
@@ -282,42 +287,70 @@ impl ast::ByteString {
 
 impl IsString for ast::CString {
     const RAW_PREFIX: &'static str = "cr";
+    const MODE: Mode = Mode::CStr;
+
+    fn escaped_char_ranges(
+        &self,
+        cb: &mut dyn FnMut(TextRange, Result<char, rustc_lexer::unescape::EscapeError>),
+    ) {
+        let text_range_no_quotes = match self.text_range_between_quotes() {
+            Some(it) => it,
+            None => return,
+        };
+
+        let start = self.syntax().text_range().start();
+        let text = &self.text()[text_range_no_quotes - start];
+        let offset = text_range_no_quotes.start() - start;
+
+        unescape_c_string(text, Self::MODE, &mut |range, unescaped_char| {
+            let text_range =
+                TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap());
+            // XXX: This method should only be used for highlighting ranges. The unescaped
+            // char/byte is not used. For simplicity, we return an arbitrary placeholder char.
+            cb(text_range + offset, unescaped_char.map(|_| ' '));
+        });
+    }
 }
 
 impl ast::CString {
-    pub fn value(&self) -> Option<Cow<'_, str>> {
+    pub fn value(&self) -> Option<Cow<'_, [u8]>> {
         if self.is_raw() {
             let text = self.text();
             let text =
                 &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
-            return Some(Cow::Borrowed(text));
+            return Some(Cow::Borrowed(text.as_bytes()));
         }
 
         let text = self.text();
         let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
 
-        let mut buf = String::new();
+        let mut buf = Vec::new();
         let mut prev_end = 0;
         let mut has_error = false;
-        unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match (
-            unescaped_char,
+        let mut char_buf = [0u8; 4];
+        let mut extend_unit = |buf: &mut Vec<u8>, unit: CStrUnit| match unit {
+            CStrUnit::Byte(b) => buf.push(b),
+            CStrUnit::Char(c) => buf.extend(c.encode_utf8(&mut char_buf).as_bytes()),
+        };
+        unescape_c_string(text, Self::MODE, &mut |char_range, unescaped| match (
+            unescaped,
             buf.capacity() == 0,
         ) {
-            (Ok(c), false) => buf.push(c),
+            (Ok(u), false) => extend_unit(&mut buf, u),
             (Ok(_), true) if char_range.len() == 1 && char_range.start == prev_end => {
                 prev_end = char_range.end
             }
-            (Ok(c), true) => {
+            (Ok(u), true) => {
                 buf.reserve_exact(text.len());
-                buf.push_str(&text[..prev_end]);
-                buf.push(c);
+                buf.extend(text[..prev_end].as_bytes());
+                extend_unit(&mut buf, u);
             }
             (Err(_), _) => has_error = true,
         });
 
         match (has_error, buf.capacity() == 0) {
             (true, _) => None,
-            (false, true) => Some(Cow::Borrowed(text)),
+            (false, true) => Some(Cow::Borrowed(text.as_bytes())),
             (false, false) => Some(Cow::Owned(buf)),
         }
     }