language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-08-18 17:40:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

internal: Bump rustc_lexer

Browse source
This commit is contained in:
Lukas Wirth 2023-05-18 10:17:40 +02:00
parent 0a806fe7ad
commit 099b5b3b15
26 changed files with 134 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/parser/Cargo.toml
View file

@ -13,7 +13,7 @@ doctest = false
[dependencies]
drop_bomb = "0.1.5"
rustc_lexer = { version = "727.0.0", package = "rustc-ap-rustc_lexer" }
rustc_lexer.workspace = true
limit.workspace = true

56
crates/parser/src/lexed_str.rs
View file

@ -36,7 +36,7 @@ impl<'a> LexedStr<'a> {
};
for token in rustc_lexer::tokenize(&text[conv.offset..]) {
let token_text = &text[conv.offset..][..token.len];
let token_text = &text[conv.offset..][..token.len as usize];
conv.extend_token(&token.kind, token_text);
}
@ -49,8 +49,8 @@ impl<'a> LexedStr<'a> {
return None;
}
let token = rustc_lexer::first_token(text);
if token.len != text.len() {
let token = rustc_lexer::tokenize(text).next()?;
if token.len as usize != text.len() {
return None;
}
@ -175,6 +175,10 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::Ident => {
SyntaxKind::from_keyword(token_text).unwrap_or(IDENT)
}
rustc_lexer::TokenKind::InvalidIdent => {
err = "Ident contains invalid characters";
IDENT
}
rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => {
@ -221,6 +225,7 @@ impl<'a> Converter<'a> {
err = "unknown literal prefix";
IDENT
}
rustc_lexer::TokenKind::Eof => EOF,
}
};
@ -268,35 +273,30 @@ impl<'a> Converter<'a> {
}
BYTE_STRING
}
rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => {
if let Some(raw_str_err) = raw_str_err {
err = match raw_str_err {
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal",
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
"Missing trailing `\"` to terminate the raw string literal"
} else {
"Missing trailing `\"` with `#` symbols to terminate the raw string literal"
},
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols",
};
};
rustc_lexer::LiteralKind::CStr { terminated } => {
if !terminated {
err = "Missing trailing `\"` symbol to terminate the string literal";
}
STRING
}
rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => {
if let Some(raw_str_err) = raw_str_err {
err = match raw_str_err {
rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal",
rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found {
"Missing trailing `\"` to terminate the raw byte string literal"
} else {
"Missing trailing `\"` with `#` symbols to terminate the raw byte string literal"
},
rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols",
};
};
rustc_lexer::LiteralKind::RawStr { n_hashes } => {
if n_hashes.is_none() {
err = "Invalid raw string literal";
}
STRING
}
rustc_lexer::LiteralKind::RawByteStr { n_hashes } => {
if n_hashes.is_none() {
err = "Invalid raw string literal";
}
BYTE_STRING
}
rustc_lexer::LiteralKind::RawCStr { n_hashes } => {
if n_hashes.is_none() {
err = "Invalid raw string literal";
}
STRING
}
};
let err = if err.is_empty() { None } else { Some(err) };

3
crates/parser/src/syntax_kind/generated.rs
View file

@ -117,6 +117,7 @@ pub enum SyntaxKind {
BYTE,
STRING,
BYTE_STRING,
C_STRING,
ERROR,
IDENT,
WHITESPACE,
@ -379,7 +380,7 @@ impl SyntaxKind {
)
}
pub fn is_literal(self) -> bool {
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING)
matches!(self, INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING | C_STRING)
}
pub fn from_keyword(ident: &str) -> Option<SyntaxKind> {
let kw = match ident {

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\"" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\x7f" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"🦀" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\n" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\" " error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\" " error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.rast
View file

@ -1 +1 @@
BYTE_STRING "br##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal
BYTE_STRING "br##\"\\u{20AA}" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.rast
View file

@ -1 +1 @@
STRING "r##\"" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.rast
View file

@ -1 +1 @@
STRING "r##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\x7f" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.rast
View file

@ -1 +1 @@
STRING "r##\"🦀" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"🦀" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.rast
View file

@ -1 +1 @@
STRING "r##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.rast
View file

@ -1 +1 @@
STRING "r##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\n" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.rast
View file

@ -1 +1 @@
STRING "r##\" " error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\" " error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.rast
View file

@ -1 +1 @@
STRING "r##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw string literal
STRING "r##\"\\u{20AA}" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.rast
View file

@ -1 +1 @@
BYTE_STRING "br##" error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
BYTE_STRING "br##" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.rast
View file

@ -1,4 +1,4 @@
BYTE_STRING "br## " error: Missing `"` symbol after `#` symbols to begin the raw byte string literal
BYTE_STRING "br## " error: Invalid raw string literal
IDENT "I"
WHITESPACE " "
IDENT "lack"

2
crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.rast
View file

@ -1 +1 @@
STRING "r##" error: Missing `"` symbol after `#` symbols to begin the raw string literal
STRING "r##" error: Invalid raw string literal

2
crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.rast
View file

@ -1,4 +1,4 @@
STRING "r## " error: Missing `"` symbol after `#` symbols to begin the raw string literal
STRING "r## " error: Invalid raw string literal
IDENT "I"
WHITESPACE " "
IDENT "lack"