language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-30 05:45:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Fix incorrect encoding of literals in the proc-macro-api on version 4

Browse source
This commit is contained in:
Lukas Wirth 2024-07-15 14:41:35 +02:00
parent f913901399
commit 05ce57efd5
12 changed files with 183 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

9
crates/mbe/src/lib.rs
View file

@ -6,13 +6,6 @@
//! The tests for this functionality live in another crate:
//! `hir_def::macro_expansion_tests::mbe`.
#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
#[cfg(not(feature = "in-rust-tree"))]
extern crate ra_ap_rustc_lexer as rustc_lexer;
#[cfg(feature = "in-rust-tree")]
extern crate rustc_lexer;
mod expander;
mod parser;
mod syntax_bridge;
@ -36,7 +29,7 @@ pub use tt::{Delimiter, DelimiterKind, Punct};
pub use crate::syntax_bridge::{
desugar_doc_comment_text, parse_exprs_with_sep, parse_to_token_tree,
parse_to_token_tree_static_span, syntax_node_to_token_tree, syntax_node_to_token_tree_modified,
token_to_literal, token_tree_to_syntax_node, DocCommentDesugarMode, SpanMapper,
token_tree_to_syntax_node, DocCommentDesugarMode, SpanMapper,
};
pub use crate::syntax_bridge::dummy_test_span_utils::*;

53
crates/mbe/src/syntax_bridge.rs
View file

@ -4,7 +4,7 @@ use std::fmt;
use rustc_hash::{FxHashMap, FxHashSet};
use span::{Edition, SpanAnchor, SpanData, SpanMap};
use stdx::{format_to, itertools::Itertools, never, non_empty_vec::NonEmptyVec};
use stdx::{format_to, never, non_empty_vec::NonEmptyVec};
use syntax::{
ast::{self, make::tokens::doc_comment},
format_smolstr, AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement,
@ -14,6 +14,7 @@ use syntax::{
use tt::{
buffer::{Cursor, TokenBuffer},
iter::TtIter,
token_to_literal,
};
use crate::to_parser_input::to_parser_input;
@ -400,56 +401,6 @@ where
}
}
pub fn token_to_literal<S>(text: SmolStr, span: S) -> tt::Literal<S>
where
S: Copy,
{
use rustc_lexer::LiteralKind;
let token = rustc_lexer::tokenize(&text).next_tuple();
let Some((rustc_lexer::Token {
kind: rustc_lexer::TokenKind::Literal { kind, suffix_start },
..
},)) = token
else {
return tt::Literal { span, text, kind: tt::LitKind::Err(()), suffix: None };
};
let (kind, start_offset, end_offset) = match kind {
LiteralKind::Int { .. } => (tt::LitKind::Integer, 0, 0),
LiteralKind::Float { .. } => (tt::LitKind::Float, 0, 0),
LiteralKind::Char { terminated } => (tt::LitKind::Char, 1, terminated as usize),
LiteralKind::Byte { terminated } => (tt::LitKind::Byte, 2, terminated as usize),
LiteralKind::Str { terminated } => (tt::LitKind::Str, 1, terminated as usize),
LiteralKind::ByteStr { terminated } => (tt::LitKind::ByteStr, 2, terminated as usize),
LiteralKind::CStr { terminated } => (tt::LitKind::CStr, 2, terminated as usize),
LiteralKind::RawStr { n_hashes } => (
tt::LitKind::StrRaw(n_hashes.unwrap_or_default()),
2 + n_hashes.unwrap_or_default() as usize,
1 + n_hashes.unwrap_or_default() as usize,
),
LiteralKind::RawByteStr { n_hashes } => (
tt::LitKind::ByteStrRaw(n_hashes.unwrap_or_default()),
3 + n_hashes.unwrap_or_default() as usize,
1 + n_hashes.unwrap_or_default() as usize,
),
LiteralKind::RawCStr { n_hashes } => (
tt::LitKind::CStrRaw(n_hashes.unwrap_or_default()),
3 + n_hashes.unwrap_or_default() as usize,
1 + n_hashes.unwrap_or_default() as usize,
),
};
let (lit, suffix) = text.split_at(suffix_start as usize);
let lit = &lit[start_offset..lit.len() - end_offset];
let suffix = match suffix {
"" | "_" => None,
suffix => Some(Box::new(suffix.into())),
};
tt::Literal { span, text: lit.into(), kind, suffix }
}
fn is_single_token_op(kind: SyntaxKind) -> bool {
matches!(
kind,