language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-10-03 15:15:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

internal: Improve parser recovery a bunch

Browse source
This commit is contained in:
Lukas Wirth 2023-02-11 18:06:07 +01:00
parent 5fdf640fb3
commit 1be24e0899
14 changed files with 249 additions and 134 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -1173,6 +1173,7 @@ dependencies = [
"limit", "limit",
"rustc-ap-rustc_lexer", "rustc-ap-rustc_lexer",
"sourcegen", "sourcegen",
"stdx",
] ]
[[package]] [[package]]

2
crates/hir-def/src/macro_expansion_tests/mbe.rs
View file

@ -1476,7 +1476,7 @@ macro_rules! m {
/* parse error: expected identifier */ /* parse error: expected identifier */
/* parse error: expected SEMICOLON */ /* parse error: expected SEMICOLON */
/* parse error: expected SEMICOLON */ /* parse error: expected SEMICOLON */
/* parse error: expected expression */ /* parse error: expected expression, item or let statement */
fn f() { fn f() {
K::(C("0")); K::(C("0"));
} }

2
crates/hir-def/src/macro_expansion_tests/mbe/regression.rs
View file

@ -831,7 +831,7 @@ macro_rules! rgb_color {
/* parse error: expected R_ANGLE */ /* parse error: expected R_ANGLE */
/* parse error: expected SEMICOLON */ /* parse error: expected SEMICOLON */
/* parse error: expected SEMICOLON */ /* parse error: expected SEMICOLON */
/* parse error: expected expression */ /* parse error: expected expression, item or let statement */
pub fn new() { pub fn new() {
let _ = 0as u32<<(8+8); let _ = 0as u32<<(8+8);
} }

1
crates/parser/Cargo.toml
View file

@ -20,4 +20,5 @@ limit.workspace = true
[dev-dependencies] [dev-dependencies]
expect-test = "1.4.0" expect-test = "1.4.0"
stdx.workspace = true
sourcegen.workspace = true sourcegen.workspace = true

29
crates/parser/src/grammar/expressions.rs
View file

@ -68,6 +68,12 @@ pub(super) fn stmt(p: &mut Parser<'_>, semicolon: Semicolon) {
Err(m) => m, Err(m) => m,
}; };
if !p.at_ts(EXPR_FIRST) {
p.err_and_bump("expected expression, item or let statement");
m.abandon(p);
return;
}
if let Some((cm, blocklike)) = expr_stmt(p, Some(m)) { if let Some((cm, blocklike)) = expr_stmt(p, Some(m)) {
if !(p.at(T!['}']) || (semicolon != Semicolon::Required && p.at(EOF))) { if !(p.at(T!['}']) || (semicolon != Semicolon::Required && p.at(EOF))) {
// test no_semi_after_block // test no_semi_after_block
@ -227,6 +233,12 @@ fn expr_bp(
attributes::outer_attrs(p); attributes::outer_attrs(p);
m m
}); });
if !p.at_ts(EXPR_FIRST) {
p.err_recover("expected expression", atom::EXPR_RECOVERY_SET);
m.abandon(p);
return None;
}
let mut lhs = match lhs(p, r) { let mut lhs = match lhs(p, r) {
Some((lhs, blocklike)) => { Some((lhs, blocklike)) => {
let lhs = lhs.extend_to(p, m); let lhs = lhs.extend_to(p, m);
@ -551,6 +563,12 @@ fn cast_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
m.complete(p, CAST_EXPR) m.complete(p, CAST_EXPR)
} }
// test_err arg_list_recovery
// fn main() {
// foo(bar::);
// foo(bar:);
// foo(bar+);
// }
fn arg_list(p: &mut Parser<'_>) { fn arg_list(p: &mut Parser<'_>) {
assert!(p.at(T!['('])); assert!(p.at(T!['(']));
let m = p.start(); let m = p.start();
@ -563,8 +581,15 @@ fn arg_list(p: &mut Parser<'_>) {
if !expr(p) { if !expr(p) {
break; break;
} }
if !p.at(T![')']) && !p.expect(T![,]) { if !p.at(T![,]) {
break; if p.at_ts(EXPR_FIRST) {
p.error("expected `,`");
continue;
} else {
break;
}
} else {
p.bump(T![,]);
} }
} }
p.eat(T![')']); p.eat(T![')']);

32
crates/parser/src/grammar/expressions/atom.rs
View file

@ -40,26 +40,28 @@ pub(super) const ATOM_EXPR_FIRST: TokenSet =
T!['{'], T!['{'],
T!['['], T!['['],
T![|], T![|],
T![move],
T![box],
T![if],
T![while],
T![match],
T![unsafe],
T![return],
T![yield],
T![do],
T![break],
T![continue],
T![async], T![async],
T![try], T![box],
T![break],
T![const], T![const],
T![loop], T![continue],
T![do],
T![for], T![for],
T![if],
T![let],
T![loop],
T![match],
T![move],
T![return],
T![static],
T![try],
T![unsafe],
T![while],
T![yield],
LIFETIME_IDENT, LIFETIME_IDENT,
])); ]));
const EXPR_RECOVERY_SET: TokenSet = TokenSet::new(&[T![let]]); pub(super) const EXPR_RECOVERY_SET: TokenSet = TokenSet::new(&[T![')'], T![']']]);
pub(super) fn atom_expr( pub(super) fn atom_expr(
p: &mut Parser<'_>, p: &mut Parser<'_>,
@ -157,7 +159,7 @@ pub(super) fn atom_expr(
T![for] => for_expr(p, None), T![for] => for_expr(p, None),
_ => { _ => {
p.err_recover("expected expression", EXPR_RECOVERY_SET); p.err_and_bump("expected expression");
return None; return None;
} }
}; };

11
crates/parser/src/grammar/paths.rs
View file

@ -67,6 +67,10 @@ fn path_for_qualifier(
} }
} }
const EXPR_PATH_SEGMENT_RECOVERY_SET: TokenSet =
items::ITEM_RECOVERY_SET.union(TokenSet::new(&[T![')'], T![,], T![let]]));
const TYPE_PATH_SEGMENT_RECOVERY_SET: TokenSet = types::TYPE_RECOVERY_SET;
fn path_segment(p: &mut Parser<'_>, mode: Mode, first: bool) { fn path_segment(p: &mut Parser<'_>, mode: Mode, first: bool) {
let m = p.start(); let m = p.start();
// test qual_paths // test qual_paths
@ -102,7 +106,12 @@ fn path_segment(p: &mut Parser<'_>, mode: Mode, first: bool) {
m.complete(p, NAME_REF); m.complete(p, NAME_REF);
} }
_ => { _ => {
p.err_recover("expected identifier", items::ITEM_RECOVERY_SET); let recover_set = match mode {
Mode::Use => items::ITEM_RECOVERY_SET,
Mode::Type => TYPE_PATH_SEGMENT_RECOVERY_SET,
Mode::Expr => EXPR_PATH_SEGMENT_RECOVERY_SET,
};
p.err_recover("expected identifier", recover_set);
if empty { if empty {
// test_err empty_segment // test_err empty_segment
// use crate::; // use crate::;

3
crates/parser/src/grammar/types.rs
View file

@ -17,8 +17,9 @@ pub(super) const TYPE_FIRST: TokenSet = paths::PATH_FIRST.union(TokenSet::new(&[
T![Self], T![Self],
])); ]));
const TYPE_RECOVERY_SET: TokenSet = TokenSet::new(&[ pub(super) const TYPE_RECOVERY_SET: TokenSet = TokenSet::new(&[
T![')'], T![')'],
T![>],
T![,], T![,],
// test_err struct_field_recover // test_err struct_field_recover
// struct S { f pub g: () } // struct S { f pub g: () }

6
crates/parser/src/tests.rs
View file

@ -15,6 +15,7 @@ use crate::{LexedStr, TopEntryPoint};
#[test] #[test]
fn lex_ok() { fn lex_ok() {
for case in TestCase::list("lexer/ok") { for case in TestCase::list("lexer/ok") {
let _guard = stdx::panic_context::enter(format!("{:?}", case.rs));
let actual = lex(&case.text); let actual = lex(&case.text);
expect_file![case.rast].assert_eq(&actual) expect_file![case.rast].assert_eq(&actual)
} }
@ -23,6 +24,7 @@ fn lex_ok() {
#[test] #[test]
fn lex_err() { fn lex_err() {
for case in TestCase::list("lexer/err") { for case in TestCase::list("lexer/err") {
let _guard = stdx::panic_context::enter(format!("{:?}", case.rs));
let actual = lex(&case.text); let actual = lex(&case.text);
expect_file![case.rast].assert_eq(&actual) expect_file![case.rast].assert_eq(&actual)
} }
@ -46,6 +48,7 @@ fn lex(text: &str) -> String {
#[test] #[test]
fn parse_ok() { fn parse_ok() {
for case in TestCase::list("parser/ok") { for case in TestCase::list("parser/ok") {
let _guard = stdx::panic_context::enter(format!("{:?}", case.rs));
let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
assert!(!errors, "errors in an OK file {}:\n{actual}", case.rs.display()); assert!(!errors, "errors in an OK file {}:\n{actual}", case.rs.display());
expect_file![case.rast].assert_eq(&actual); expect_file![case.rast].assert_eq(&actual);
@ -55,6 +58,7 @@ fn parse_ok() {
#[test] #[test]
fn parse_inline_ok() { fn parse_inline_ok() {
for case in TestCase::list("parser/inline/ok") { for case in TestCase::list("parser/inline/ok") {
let _guard = stdx::panic_context::enter(format!("{:?}", case.rs));
let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
assert!(!errors, "errors in an OK file {}:\n{actual}", case.rs.display()); assert!(!errors, "errors in an OK file {}:\n{actual}", case.rs.display());
expect_file![case.rast].assert_eq(&actual); expect_file![case.rast].assert_eq(&actual);
@ -64,6 +68,7 @@ fn parse_inline_ok() {
#[test] #[test]
fn parse_err() { fn parse_err() {
for case in TestCase::list("parser/err") { for case in TestCase::list("parser/err") {
let _guard = stdx::panic_context::enter(format!("{:?}", case.rs));
let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
assert!(errors, "no errors in an ERR file {}:\n{actual}", case.rs.display()); assert!(errors, "no errors in an ERR file {}:\n{actual}", case.rs.display());
expect_file![case.rast].assert_eq(&actual) expect_file![case.rast].assert_eq(&actual)
@ -73,6 +78,7 @@ fn parse_err() {
#[test] #[test]
fn parse_inline_err() { fn parse_inline_err() {
for case in TestCase::list("parser/inline/err") { for case in TestCase::list("parser/inline/err") {
let _guard = stdx::panic_context::enter(format!("{:?}", case.rs));
let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text); let (actual, errors) = parse(TopEntryPoint::SourceFile, &case.text);
assert!(errors, "no errors in an ERR file {}:\n{actual}", case.rs.display()); assert!(errors, "no errors in an ERR file {}:\n{actual}", case.rs.display());
expect_file![case.rast].assert_eq(&actual) expect_file![case.rast].assert_eq(&actual)

2
crates/parser/src/tests/top_entries.rs
View file

@ -65,7 +65,7 @@ fn macro_stmt() {
MACRO_STMTS MACRO_STMTS
ERROR ERROR
SHEBANG "#!/usr/bin/rust" SHEBANG "#!/usr/bin/rust"
error 0: expected expression error 0: expected expression, item or let statement
"##]], "##]],
); );
check( check(

16
crates/parser/test_data/parser/err/0022_bad_exprs.rast
View file

@ -145,27 +145,27 @@ SOURCE_FILE
error 16: expected expression error 16: expected expression
error 17: expected R_BRACK error 17: expected R_BRACK
error 17: expected SEMICOLON error 17: expected SEMICOLON
error 17: expected expression error 17: expected expression, item or let statement
error 25: expected a name error 25: expected a name
error 26: expected `;`, `{`, or `(` error 26: expected `;`, `{`, or `(`
error 30: expected pattern error 30: expected pattern
error 31: expected SEMICOLON error 31: expected SEMICOLON
error 53: expected expression error 53: expected expression
error 54: expected SEMICOLON error 54: expected SEMICOLON
error 54: expected expression error 54: expected expression, item or let statement
error 60: expected type error 60: expected type
error 60: expected `{` error 60: expected `{`
error 60: expected expression error 60: expected expression, item or let statement
error 65: expected pattern error 65: expected pattern
error 65: expected SEMICOLON error 65: expected SEMICOLON
error 65: expected expression error 65: expected expression, item or let statement
error 92: expected expression error 92: expected expression
error 93: expected SEMICOLON error 93: expected SEMICOLON
error 93: expected expression error 93: expected expression, item or let statement
error 95: expected expression error 95: expected expression, item or let statement
error 96: expected expression error 96: expected expression, item or let statement
error 103: expected a name error 103: expected a name
error 104: expected `{` error 104: expected `{`
error 108: expected pattern error 108: expected pattern
error 108: expected SEMICOLON error 108: expected SEMICOLON
error 108: expected expression error 108: expected expression, item or let statement

196
crates/parser/test_data/parser/err/0024_many_type_parens.rast
View file

@ -180,32 +180,32 @@ SOURCE_FILE
ERROR ERROR
PLUS "+" PLUS "+"
WHITESPACE " " WHITESPACE " "
TUPLE_EXPR EXPR_STMT
L_PAREN "("
CLOSURE_EXPR
FOR_KW "for"
GENERIC_PARAM_LIST
L_ANGLE "<"
LIFETIME_PARAM
LIFETIME
LIFETIME_IDENT "'a"
R_ANGLE ">"
WHITESPACE " "
BIN_EXPR BIN_EXPR
BIN_EXPR BIN_EXPR
BIN_EXPR TUPLE_EXPR
L_PAREN "("
CLOSURE_EXPR
FOR_KW "for"
GENERIC_PARAM_LIST
L_ANGLE "<"
LIFETIME_PARAM
LIFETIME
LIFETIME_IDENT "'a"
R_ANGLE ">"
WHITESPACE " "
BIN_EXPR BIN_EXPR
PATH_EXPR BIN_EXPR
PATH PATH_EXPR
PATH_SEGMENT PATH
NAME_REF PATH_SEGMENT
IDENT "Trait" NAME_REF
L_ANGLE "<" IDENT "Trait"
ERROR L_ANGLE "<"
LIFETIME_IDENT "'a" ERROR
R_ANGLE ">" LIFETIME_IDENT "'a"
ERROR R_ANGLE ">"
R_PAREN ")" R_PAREN ")"
WHITESPACE " " WHITESPACE " "
PLUS "+" PLUS "+"
WHITESPACE " " WHITESPACE " "
@ -220,108 +220,96 @@ SOURCE_FILE
R_ANGLE ">" R_ANGLE ">"
ERROR ERROR
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n " WHITESPACE "\n "
LET_EXPR LET_STMT
LET_KW "let" LET_KW "let"
WHITESPACE " "
WILDCARD_PAT
UNDERSCORE "_"
ERROR
COLON ":"
WHITESPACE " " WHITESPACE " "
BIN_EXPR WILDCARD_PAT
BIN_EXPR UNDERSCORE "_"
PATH_EXPR COLON ":"
PATH WHITESPACE " "
PATH_SEGMENT DYN_TRAIT_TYPE
NAME_REF TYPE_BOUND_LIST
IDENT "Box" TYPE_BOUND
L_ANGLE "<" PATH_TYPE
TUPLE_EXPR PATH
L_PAREN "(" PATH_SEGMENT
CLOSURE_EXPR NAME_REF
FOR_KW "for" IDENT "Box"
GENERIC_PARAM_LIST GENERIC_ARG_LIST
L_ANGLE "<"
LIFETIME_PARAM
LIFETIME
LIFETIME_IDENT "'a"
R_ANGLE ">"
WHITESPACE " "
BIN_EXPR
BIN_EXPR
BIN_EXPR
BIN_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "Trait"
L_ANGLE "<" L_ANGLE "<"
ERROR TYPE_ARG
LIFETIME_IDENT "'a" PAREN_TYPE
R_ANGLE ">" L_PAREN "("
ERROR FOR_TYPE
R_PAREN ")" FOR_KW "for"
WHITESPACE " " GENERIC_PARAM_LIST
PLUS "+" L_ANGLE "<"
WHITESPACE " " LIFETIME_PARAM
PAREN_EXPR LIFETIME
L_PAREN "(" LIFETIME_IDENT "'a"
PATH_EXPR R_ANGLE ">"
PATH WHITESPACE " "
PATH_SEGMENT PATH_TYPE
NAME_REF PATH
IDENT "Copy" PATH_SEGMENT
R_PAREN ")" NAME_REF
WHITESPACE " " IDENT "Trait"
PLUS "+" GENERIC_ARG_LIST
WHITESPACE " " L_ANGLE "<"
PAREN_EXPR LIFETIME_ARG
L_PAREN "(" LIFETIME
ERROR LIFETIME_IDENT "'a"
QUESTION "?" R_ANGLE ">"
PATH_EXPR R_PAREN ")"
WHITESPACE " "
PLUS "+"
WHITESPACE " "
TYPE_BOUND
L_PAREN "("
PATH_TYPE
PATH
PATH_SEGMENT
NAME_REF
IDENT "Copy"
R_PAREN ")"
WHITESPACE " "
PLUS "+"
WHITESPACE " "
TYPE_BOUND
L_PAREN "("
QUESTION "?"
PATH_TYPE
PATH PATH
PATH_SEGMENT PATH_SEGMENT
NAME_REF NAME_REF
IDENT "Sized" IDENT "Sized"
R_PAREN ")" R_PAREN ")"
R_ANGLE ">" ERROR
ERROR R_ANGLE ">"
SEMICOLON ";" SEMICOLON ";"
WHITESPACE "\n" WHITESPACE "\n"
R_CURLY "}" R_CURLY "}"
WHITESPACE "\n" WHITESPACE "\n"
error 88: expected COMMA error 88: expected COMMA
error 88: expected R_ANGLE error 88: expected R_ANGLE
error 121: expected SEMICOLON error 121: expected SEMICOLON
error 121: expected expression error 121: expected expression, item or let statement
error 140: expected type error 140: expected type
error 141: expected R_PAREN error 141: expected R_PAREN
error 141: expected COMMA error 141: expected COMMA
error 141: expected R_ANGLE error 141: expected R_ANGLE
error 141: expected SEMICOLON error 141: expected SEMICOLON
error 146: expected SEMICOLON error 146: expected SEMICOLON
error 146: expected expression error 146: expected expression, item or let statement
error 148: expected expression error 148: expected expression, item or let statement
error 158: expected `|` error 158: expected `|`
error 158: expected COMMA error 158: expected COMMA
error 165: expected expression error 165: expected expression
error 168: expected expression error 168: expected expression
error 179: expected expression error 179: expected expression
error 180: expected COMMA error 180: expected SEMICOLON
error 190: expected EQ error 215: expected COMMA
error 190: expected expression error 215: expected R_ANGLE
error 191: expected COMMA error 235: expected SEMICOLON
error 204: expected `|` error 235: expected expression, item or let statement
error 204: expected COMMA
error 211: expected expression
error 214: expected expression
error 228: expected expression
error 229: expected R_PAREN
error 229: expected COMMA
error 236: expected expression
error 237: expected COMMA
error 237: expected expression
error 237: expected R_PAREN

77
crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast Normal file
View file

@ -0,0 +1,77 @@
SOURCE_FILE
FN
FN_KW "fn"
WHITESPACE " "
NAME
IDENT "main"
PARAM_LIST
L_PAREN "("
R_PAREN ")"
WHITESPACE " "
BLOCK_EXPR
STMT_LIST
L_CURLY "{"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "foo"
ARG_LIST
L_PAREN "("
PATH_EXPR
PATH
PATH
PATH_SEGMENT
NAME_REF
IDENT "bar"
COLON2 "::"
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "foo"
ARG_LIST
L_PAREN "("
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "bar"
ERROR
COLON ":"
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n "
EXPR_STMT
CALL_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "foo"
ARG_LIST
L_PAREN "("
BIN_EXPR
PATH_EXPR
PATH
PATH_SEGMENT
NAME_REF
IDENT "bar"
PLUS "+"
R_PAREN ")"
SEMICOLON ";"
WHITESPACE "\n"
R_CURLY "}"
WHITESPACE "\n"
error 25: expected identifier
error 39: expected `,`
error 39: expected expression
error 55: expected expression

5
crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rs Normal file
View file

@ -0,0 +1,5 @@
fn main() {
foo(bar::);
foo(bar:);
foo(bar+);
}