From 2d5d16f18c6dde52a53071c604b8cc30739df7be Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Tue, 3 May 2022 16:47:33 +0200 Subject: [PATCH 1/9] Remove `ast::Literal::token` --- crates/ide-completion/src/completions/postfix.rs | 4 ++-- crates/ide-completion/src/patterns.rs | 2 +- crates/syntax/src/ast/expr_ext.rs | 13 +++++++------ crates/syntax/src/validation.rs | 11 +++++++++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/crates/ide-completion/src/completions/postfix.rs b/crates/ide-completion/src/completions/postfix.rs index 7b936f0ff7..cc2a443700 100644 --- a/crates/ide-completion/src/completions/postfix.rs +++ b/crates/ide-completion/src/completions/postfix.rs @@ -5,7 +5,7 @@ mod format_like; use hir::{Documentation, HasAttrs}; use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap}; use syntax::{ - ast::{self, AstNode, AstToken}, + ast::{self, AstNode, LiteralKind}, SyntaxKind::{EXPR_STMT, STMT_LIST}, TextRange, TextSize, }; @@ -191,7 +191,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) { } if let ast::Expr::Literal(literal) = dot_receiver.clone() { - if let Some(literal_text) = ast::String::cast(literal.token()) { + if let LiteralKind::String(literal_text) = literal.kind() { add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text); } } diff --git a/crates/ide-completion/src/patterns.rs b/crates/ide-completion/src/patterns.rs index 6fdec78385..c1db8ce4a6 100644 --- a/crates/ide-completion/src/patterns.rs +++ b/crates/ide-completion/src/patterns.rs @@ -230,7 +230,7 @@ pub(crate) fn determine_location( let receiver = find_in_original_file(it.expr(), original_file); let receiver_is_ambiguous_float_literal = if let Some(ast::Expr::Literal(l)) = &receiver { match l.kind() { - ast::LiteralKind::FloatNumber { .. } => l.token().text().ends_with('.'), + ast::LiteralKind::FloatNumber { .. } => l.to_string().ends_with('.'), _ => false, } } else { diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 17785152bc..23b1145a77 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -8,7 +8,7 @@ use crate::{ operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp}, support, AstChildren, AstNode, }, - AstToken, + AstToken, SyntaxElement, SyntaxKind::*, SyntaxNode, SyntaxToken, T, }; @@ -289,16 +289,17 @@ pub enum LiteralKind { } impl ast::Literal { - pub fn token(&self) -> SyntaxToken { + pub fn value(&self) -> SyntaxElement { self.syntax() .children_with_tokens() .find(|e| e.kind() != ATTR && !e.kind().is_trivia()) - .and_then(|e| e.into_token()) .unwrap() } - pub fn kind(&self) -> LiteralKind { - let token = self.token(); + let token = match self.value() { + rowan::NodeOrToken::Node(_node) => unreachable!(), + rowan::NodeOrToken::Token(token) => token, + }; if let Some(t) = ast::IntNumber::cast(token.clone()) { return LiteralKind::IntNumber(t); @@ -364,7 +365,7 @@ impl ast::BlockExpr { fn test_literal_with_attr() { let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#); let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap(); - assert_eq!(lit.token().text(), r#""Hello""#); + assert_eq!(lit.value().to_string(), r#""Hello""#); } impl ast::RecordExprField { diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs index c2c2c82e11..3edca3eb8f 100644 --- a/crates/syntax/src/validation.rs +++ b/crates/syntax/src/validation.rs @@ -119,8 +119,15 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec) { text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end)) } - let token = literal.token(); - let text = token.text(); + let token = literal.value(); + let text; + let text = match &token { + rowan::NodeOrToken::Node(node) => { + text = node.text().to_string(); + &*text + } + rowan::NodeOrToken::Token(token) => token.text(), + }; // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| { From 502c519e7d15ca6d015b4ca48951e43e893c8cba Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Tue, 3 May 2022 19:56:10 +0200 Subject: [PATCH 2/9] Wrap float literals in their own node --- crates/hir-def/src/body/lower.rs | 2 +- crates/hir-expand/src/builtin_fn_macro.rs | 36 ++++++++----------- crates/parser/src/grammar/expressions/atom.rs | 8 ++++- crates/parser/src/syntax_kind/generated.rs | 3 +- .../parser/inline/ok/0085_expr_literals.rast | 3 +- .../test_data/parser/ok/0056_neq_in_type.rast | 6 ++-- crates/syntax/rust.ungram | 5 ++- crates/syntax/src/ast/expr_ext.rs | 31 +++++++++++++--- crates/syntax/src/ast/generated/nodes.rs | 26 ++++++++++++++ crates/syntax/src/ast/make.rs | 13 ++++--- crates/syntax/src/ast/token_ext.rs | 27 +++++++++----- crates/syntax/src/tests/ast_src.rs | 1 + crates/syntax/src/tests/sourcegen_ast.rs | 1 + 13 files changed, 113 insertions(+), 49 deletions(-) diff --git a/crates/hir-def/src/body/lower.rs b/crates/hir-def/src/body/lower.rs index e8303ec40f..84662aa277 100644 --- a/crates/hir-def/src/body/lower.rs +++ b/crates/hir-def/src/body/lower.rs @@ -972,7 +972,7 @@ impl From for Literal { } } LiteralKind::FloatNumber(lit) => { - let ty = lit.suffix().and_then(BuiltinFloat::from_suffix); + let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s)); Literal::Float(Default::default(), ty) } LiteralKind::ByteString(bs) => { diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 8d2352f06e..45eb660a6c 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -4,10 +4,7 @@ use base_db::{AnchoredPath, Edition, FileId}; use cfg::CfgExpr; use either::Either; use mbe::{parse_exprs_with_sep, parse_to_token_tree}; -use syntax::{ - ast::{self, AstToken}, - SmolStr, -}; +use syntax::{ast, SmolStr}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; @@ -358,14 +355,7 @@ fn unreachable_expand( } fn unquote_str(lit: &tt::Literal) -> Option { - let lit = ast::make::tokens::literal(&lit.to_string()); - let token = ast::String::cast(lit)?; - token.value().map(|it| it.into_owned()) -} - -fn unquote_byte_string(lit: &tt::Literal) -> Option> { - let lit = ast::make::tokens::literal(&lit.to_string()); - let token = ast::ByteString::cast(lit)?; + let token = ast::make::literal(&lit.to_string()).as_string()?; token.value().map(|it| it.into_owned()) } @@ -442,12 +432,16 @@ fn concat_bytes_expand( for (i, t) in tt.token_trees.iter().enumerate() { match t { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let token = ast::make::tokens::literal(&lit.to_string()); - match token.kind() { - syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()), - syntax::SyntaxKind::BYTE_STRING => { - let components = unquote_byte_string(lit).unwrap_or_else(Vec::new); - components.into_iter().for_each(|x| bytes.push(x.to_string())); + let lit = ast::make::literal(&lit.to_string()); + match lit.kind() { + ast::LiteralKind::ByteString(s) => { + s.value() + .unwrap_or_default() + .into_iter() + .for_each(|x| bytes.push(x.to_string())); + } + ast::LiteralKind::Byte => { + bytes.push(lit.to_string()); } _ => { err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); @@ -481,10 +475,10 @@ fn concat_bytes_expand_subtree( for (ti, tt) in tree.token_trees.iter().enumerate() { match tt { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let lit = ast::make::tokens::literal(&lit.to_string()); + let lit = ast::make::literal(&lit.to_string()); match lit.kind() { - syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => { - bytes.push(lit.text().to_string()) + ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte => { + bytes.push(lit.to_string()); } _ => { return Err(mbe::ExpandError::UnexpectedToken.into()); diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 10e5d897e0..37f8a7e3b7 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -29,7 +29,13 @@ pub(crate) fn literal(p: &mut Parser) -> Option { return None; } let m = p.start(); - p.bump_any(); + if p.at(FLOAT_NUMBER) { + let f = p.start(); + p.bump(FLOAT_NUMBER); + f.complete(p, FLOAT_LITERAL); + } else { + p.bump_any(); + } Some(m.complete(p, LITERAL)) } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 628fa745e7..47bf4ba92b 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -227,6 +227,7 @@ pub enum SyntaxKind { PATH, PATH_SEGMENT, LITERAL, + FLOAT_LITERAL, RENAME, VISIBILITY, WHERE_CLAUSE, @@ -386,5 +387,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number] => { $ crate :: SyntaxKind :: FLOAT_NUMBER } ; } pub use T; diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index 403c265ea3..065af27f10 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -57,7 +57,8 @@ SOURCE_FILE EQ "=" WHITESPACE " " LITERAL - FLOAT_NUMBER "2.0" + FLOAT_LITERAL + FLOAT_NUMBER "2.0" SEMICOLON ";" WHITESPACE "\n " LET_STMT diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index 55ce31275f..903a1507d0 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -19,7 +19,8 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_LITERAL + FLOAT_NUMBER "1.0f32" DOT "." NAME_REF IDENT "floor" @@ -40,7 +41,8 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_LITERAL + FLOAT_NUMBER "1.0f32" DOT "." NAME_REF IDENT "floor" diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 62aa478399..4b3e483bff 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -365,13 +365,16 @@ MacroExpr = Literal = Attr* value:( - 'int_number' | 'float_number' + 'int_number' | FloatLiteral | 'string' | 'raw_string' | 'byte_string' | 'raw_byte_string' | 'true' | 'false' | 'char' | 'byte' ) +FloatLiteral = + 'float_number' + PathExpr = Attr* Path diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 23b1145a77..f4bff80c4c 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -282,7 +282,7 @@ pub enum LiteralKind { String(ast::String), ByteString(ast::ByteString), IntNumber(ast::IntNumber), - FloatNumber(ast::FloatNumber), + FloatNumber(ast::FloatLiteral), Char(ast::Char), Byte(ast::Byte), Bool(bool), @@ -297,16 +297,17 @@ impl ast::Literal { } pub fn kind(&self) -> LiteralKind { let token = match self.value() { - rowan::NodeOrToken::Node(_node) => unreachable!(), + rowan::NodeOrToken::Node(node) => { + return LiteralKind::FloatNumber( + ast::FloatLiteral::cast(node).expect("unreachable"), + ); + } rowan::NodeOrToken::Token(token) => token, }; if let Some(t) = ast::IntNumber::cast(token.clone()) { return LiteralKind::IntNumber(t); } - if let Some(t) = ast::FloatNumber::cast(token.clone()) { - return LiteralKind::FloatNumber(t); - } if let Some(t) = ast::String::cast(token.clone()) { return LiteralKind::String(t); } @@ -326,6 +327,26 @@ impl ast::Literal { _ => unreachable!(), } } + + pub fn as_string(&self) -> Option { + match self.kind() { + LiteralKind::String(it) => Some(it), + _ => None, + } + } + + pub fn as_byte_string(&self) -> Option { + match self.kind() { + LiteralKind::ByteString(it) => Some(it), + _ => None, + } + } +} + +impl ast::FloatLiteral { + pub fn suffix(&self) -> Option { + ast::FloatNumber::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) + } } pub enum BlockModifier { diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index cf90ba64cf..b294849937 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1085,6 +1085,16 @@ impl UnderscoreExpr { pub fn underscore_token(&self) -> Option { support::token(&self.syntax, T![_]) } } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct FloatLiteral { + pub(crate) syntax: SyntaxNode, +} +impl FloatLiteral { + pub fn float_number_token(&self) -> Option { + support::token(&self.syntax, T![float_number]) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct StmtList { pub(crate) syntax: SyntaxNode, @@ -2719,6 +2729,17 @@ impl AstNode for UnderscoreExpr { } fn syntax(&self) -> &SyntaxNode { &self.syntax } } +impl AstNode for FloatLiteral { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL } + fn cast(syntax: SyntaxNode) -> Option { + if Self::can_cast(syntax.kind()) { + Some(Self { syntax }) + } else { + None + } + } + fn syntax(&self) -> &SyntaxNode { &self.syntax } +} impl AstNode for StmtList { fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST } fn cast(syntax: SyntaxNode) -> Option { @@ -4608,6 +4629,11 @@ impl std::fmt::Display for UnderscoreExpr { std::fmt::Display::fmt(self.syntax(), f) } } +impl std::fmt::Display for FloatLiteral { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(self.syntax(), f) + } +} impl std::fmt::Display for StmtList { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) diff --git a/crates/syntax/src/ast/make.rs b/crates/syntax/src/ast/make.rs index 5908dda8e6..846f4f3c71 100644 --- a/crates/syntax/src/ast/make.rs +++ b/crates/syntax/src/ast/make.rs @@ -799,6 +799,11 @@ pub fn struct_( )) } +pub fn literal(text: &str) -> ast::Literal { + assert_eq!(text.trim(), text); + ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)) +} + #[track_caller] fn ast_from_text(text: &str) -> N { let parse = SourceFile::parse(text); @@ -827,7 +832,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken { pub mod tokens { use once_cell::sync::Lazy; - use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; + use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; pub(super) static SOURCE_FILE: Lazy> = Lazy::new(|| { SourceFile::parse( @@ -858,12 +863,6 @@ pub mod tokens { sf.syntax().first_child_or_token().unwrap().into_token().unwrap() } - pub fn literal(text: &str) -> SyntaxToken { - assert_eq!(text.trim(), text); - let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)); - lit.syntax().first_child_or_token().unwrap().into_token().unwrap() - } - pub fn single_newline() -> SyntaxToken { let res = SOURCE_FILE .tree() diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 4b6dc236b5..3063396b44 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -355,14 +355,24 @@ impl Radix { #[cfg(test)] mod tests { - use crate::ast::{self, make, FloatNumber, IntNumber}; + use crate::ast::{self, make}; fn check_float_suffix<'a>(lit: &str, expected: impl Into>) { - assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); + let suffix = match make::literal(lit).kind() { + ast::LiteralKind::FloatNumber(f) => f.suffix(), + // `1f32` lexes as an INT_NUMBER + ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()), + e => unreachable!("{e:?}"), + }; + assert_eq!(suffix.as_deref(), expected.into()); } fn check_int_suffix<'a>(lit: &str, expected: impl Into>) { - assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); + let i = match make::literal(lit).kind() { + ast::LiteralKind::IntNumber(i) => i, + _ => unreachable!(), + }; + assert_eq!(i.suffix(), expected.into()); } #[test] @@ -390,12 +400,11 @@ mod tests { } fn check_string_value<'a>(lit: &str, expected: impl Into>) { - assert_eq!( - ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) } - .value() - .as_deref(), - expected.into() - ); + let s = match make::literal(&format!("\"{}\"", lit)).kind() { + ast::LiteralKind::String(s) => s, + _ => unreachable!(), + }; + assert_eq!(s.value().as_deref(), expected.into()); } #[test] diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 2f6932a1ad..964bd6c8b3 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -183,6 +183,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "PATH", "PATH_SEGMENT", "LITERAL", + "FLOAT_LITERAL", "RENAME", "VISIBILITY", "WHERE_CLAUSE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index 4cfb8075cb..e18f575e37 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -462,6 +462,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; + [float_number] => { $crate::SyntaxKind::FLOAT_NUMBER }; } pub use T; }; From 1bc3305d9596d790817b388cd62ecdc7572c912f Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 4 May 2022 16:51:05 +0200 Subject: [PATCH 3/9] Split float literal tokens at the `.` --- .../ide/src/syntax_highlighting/highlight.rs | 2 +- .../test_data/highlight_strings.html | 14 +++--- crates/mbe/src/syntax_bridge.rs | 29 ++++++++++++ crates/mbe/src/to_parser_input.rs | 16 +++---- crates/parser/src/grammar.rs | 2 +- crates/parser/src/grammar/expressions.rs | 5 +-- crates/parser/src/grammar/expressions/atom.rs | 14 ++++-- crates/parser/src/grammar/patterns.rs | 2 +- crates/parser/src/lexed_str.rs | 23 ++++++++-- crates/parser/src/syntax_kind/generated.rs | 6 +-- .../test_data/lexer/err/empty_exponent.rast | 44 +++++++++++++------ crates/parser/test_data/lexer/ok/numbers.rast | 25 +++++++---- .../parser/err/0023_mismatched_paren.rast | 4 +- .../parser/inline/ok/0085_expr_literals.rast | 4 +- .../test_data/parser/ok/0056_neq_in_type.rast | 8 +++- crates/syntax/rust.ungram | 4 +- crates/syntax/src/ast/expr_ext.rs | 2 +- crates/syntax/src/ast/generated/nodes.rs | 5 ++- crates/syntax/src/ast/generated/tokens.rs | 8 ++-- crates/syntax/src/ast/node_ext.rs | 4 +- crates/syntax/src/ast/token_ext.rs | 2 +- crates/syntax/src/tests/ast_src.rs | 2 +- crates/syntax/src/tests/sourcegen_ast.rs | 4 +- 23 files changed, 157 insertions(+), 72 deletions(-) diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index e04fd5a7c7..f061e8637e 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -30,7 +30,7 @@ pub(super) fn token(sema: &Semantics, token: SyntaxToken) -> Optio INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { SymbolKind::Field.into() } - INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(), + INT_NUMBER | FLOAT_NUMBER_PART => HlTag::NumericLiteral.into(), BYTE => HlTag::ByteLiteral.into(), CHAR => HlTag::CharLiteral.into(), IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index 60bc290121..329184730e 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd println!("Hello {:05}!", 5); println!("Hello {:05}!", -5); println!("{:#010x}!", 27); - println!("Hello {0} is {1:.5}", "x", 0.01); - println!("Hello {1} is {2:.0$}", 5, "x", 0.01); - println!("Hello {0} is {2:.1$}", "x", 5, 0.01); - println!("Hello {} is {:.*}", "x", 5, 0.01); - println!("Hello {} is {2:.*}", "x", 5, 0.01); - println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); - println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); + println!("Hello {0} is {1:.5}", "x", 0.01); + println!("Hello {1} is {2:.0$}", 5, "x", 0.01); + println!("Hello {0} is {2:.1$}", "x", 5, 0.01); + println!("Hello {} is {:.*}", "x", 5, 0.01); + println!("Hello {} is {2:.*}", "x", 5, 0.01); + println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); + println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); println!("{}, `{name:.*}` has 3 characters", "Hello", 3, name="1234.56"); println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); println!("Hello {{}}"); diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 21a0aa4284..8b8577986d 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -260,6 +260,35 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), + FLOAT_NUMBER_PART => { + // Reassemble a split-up float token. + let mut range = range; + let mut text = token.to_text(conv).to_string(); + if let Some(dot) = conv.peek() { + if dot.kind(conv) == DOT { + let (_, dot_range) = conv.bump().unwrap(); + text += &*dot.to_text(conv); + range = TextRange::new(range.start(), dot_range.end()); + + if let Some(tail) = conv.peek() { + if tail.kind(conv) == FLOAT_NUMBER_PART { + let (_, tail_range) = conv.bump().unwrap(); + text += &*tail.to_text(conv); + range = TextRange::new(range.start(), tail_range.end()); + } + } + } + } + + result.push( + tt::Leaf::from(tt::Literal { + id: conv.id_alloc().alloc(range, synth_id), + text: text.into(), + }) + .into(), + ); + continue; + } k if k.is_literal() => make_leaf!(Literal), LIFETIME_IDENT => { let char_unit = TextSize::of('\''); diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 6faa147218..958f643320 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -35,15 +35,13 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input { let is_negated = lit.text.starts_with('-'); let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; - let kind = parser::LexedStr::single_token(inner_text) - .map(|(kind, _error)| kind) - .filter(|kind| { - kind.is_literal() - && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) - }) - .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); - - res.push(kind); + let lexed_str = parser::LexedStr::new(inner_text); + if lexed_str.is_empty() { + panic!("failed to convert literal: {:?}", lit); + } + for i in 0..lexed_str.len() { + res.push(lexed_str.kind(i)); + } } tt::Leaf::Ident(ident) => match ident.text.as_ref() { "_" => res.push(T![_]), diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 4efbf9a606..45d9b2e4e0 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -318,7 +318,7 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!(p.at(IDENT) || p.at(INT_NUMBER)); + assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART)); let m = p.start(); p.bump_any(); m.complete(p, NAME_REF); diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index b063c73a9d..85f53dd237 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -475,11 +475,8 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) { + if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) { name_ref_or_index(p); - } else if p.at(FLOAT_NUMBER) { - // FIXME: How to recover and instead parse INT + T![.]? - p.bump_any(); } else { p.error("expected field name or number"); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 37f8a7e3b7..07b0a2aee5 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,7 +17,7 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER, + FLOAT_NUMBER_PART, BYTE, CHAR, STRING, @@ -29,11 +29,19 @@ pub(crate) fn literal(p: &mut Parser) -> Option { return None; } let m = p.start(); - if p.at(FLOAT_NUMBER) { + if p.at(FLOAT_NUMBER_PART) { + // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT` let f = p.start(); - p.bump(FLOAT_NUMBER); + p.bump(FLOAT_NUMBER_PART); + if p.at(DOT) { + p.bump(DOT); + if p.at(FLOAT_NUMBER_PART) { + p.bump(FLOAT_NUMBER_PART); + } + } f.complete(p, FLOAT_LITERAL); } else { + // Everything else is just one token. p.bump_any(); } Some(m.complete(p, LITERAL)) diff --git a/crates/parser/src/grammar/patterns.rs b/crates/parser/src/grammar/patterns.rs index 1f622b32e5..c16bd8d0c7 100644 --- a/crates/parser/src/grammar/patterns.rs +++ b/crates/parser/src/grammar/patterns.rs @@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option { } fn is_literal_pat_start(p: &Parser) -> bool { - p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) + p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) || p.at_ts(expressions::LITERAL_FIRST) } diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index f4b9988eac..fae5d8884b 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -177,7 +177,7 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::Literal { kind, .. } => { - self.extend_literal(token_text.len(), kind); + self.extend_literal(token_text, kind); return; } @@ -223,7 +223,7 @@ impl<'a> Converter<'a> { self.push(syntax_kind, token_text.len(), err); } - fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) { + fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { let mut err = ""; let syntax_kind = match *kind { @@ -237,7 +237,22 @@ impl<'a> Converter<'a> { if empty_exponent { err = "Missing digits after the exponent symbol"; } - FLOAT_NUMBER + + // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` + // is lexed as a float, we split floats that contain a `.` into 3 tokens. + if let Some((before, after)) = token_text.split_once('.') { + let err = if err.is_empty() { None } else { Some(err) }; + if !before.is_empty() { + self.push(FLOAT_NUMBER_PART, before.len(), None); + } + self.push(DOT, 1, None); + if !after.is_empty() { + self.push(FLOAT_NUMBER_PART, after.len(), err); + } + return; + } + + FLOAT_NUMBER_PART } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { @@ -295,6 +310,6 @@ impl<'a> Converter<'a> { }; let err = if err.is_empty() { None } else { Some(err) }; - self.push(syntax_kind, len, err); + self.push(syntax_kind, token_text.len(), err); } } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 47bf4ba92b..31fe1b2aba 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,7 +110,7 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, - FLOAT_NUMBER, + FLOAT_NUMBER_PART, CHAR, BYTE, STRING, @@ -287,7 +287,7 @@ impl SyntaxKind { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -387,5 +387,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number] => { $ crate :: SyntaxKind :: FLOAT_NUMBER } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index af03d73ced..d0a268b8c1 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,19 +30,35 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER_PART "42" +DOT "." +FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 8d13c3f610..689422933c 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,7 +4,8 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER "0." +FLOAT_NUMBER_PART "0" +DOT "." WHITESPACE " " INT_NUMBER "0z" WHITESPACE "\n" @@ -20,11 +21,13 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER "0.1279" +FLOAT_NUMBER_PART "0" +DOT "." +FLOAT_NUMBER_PART "1279" WHITESPACE " " -FLOAT_NUMBER "0e1279" +FLOAT_NUMBER_PART "0e1279" WHITESPACE " " -FLOAT_NUMBER "0E1279" +FLOAT_NUMBER_PART "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -37,7 +40,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER "0e+1" +FLOAT_NUMBER_PART "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -45,13 +48,19 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER "0.0E-2" +FLOAT_NUMBER_PART "0" +DOT "." +FLOAT_NUMBER_PART "0E-2" WHITESPACE "\n" -FLOAT_NUMBER "0___0.10000____0000e+111__" +FLOAT_NUMBER_PART "0___0" +DOT "." +FLOAT_NUMBER_PART "10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER "92.0f32" +FLOAT_NUMBER_PART "92" +DOT "." +FLOAT_NUMBER_PART "0f32" WHITESPACE " " INT_NUMBER "11__s" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index 4064a7a1ff..f57e58a4b7 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,7 +32,9 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_NUMBER "2.0" + FLOAT_NUMBER_PART "2" + DOT "." + FLOAT_NUMBER_PART "0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index 065af27f10..367aff324a 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -58,7 +58,9 @@ SOURCE_FILE WHITESPACE " " LITERAL FLOAT_LITERAL - FLOAT_NUMBER "2.0" + FLOAT_NUMBER_PART "2" + DOT "." + FLOAT_NUMBER_PART "0" SEMICOLON ";" WHITESPACE "\n " LET_STMT diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index 903a1507d0..5fd9271c62 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -20,7 +20,9 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_NUMBER_PART "1" + DOT "." + FLOAT_NUMBER_PART "0f32" DOT "." NAME_REF IDENT "floor" @@ -42,7 +44,9 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER "1.0f32" + FLOAT_NUMBER_PART "1" + DOT "." + FLOAT_NUMBER_PART "0f32" DOT "." NAME_REF IDENT "floor" diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 4b3e483bff..8e4f07d3b5 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -373,7 +373,9 @@ Literal = ) FloatLiteral = - 'float_number' + 'float_number_part' + '.'? + 'float_number_part'? PathExpr = Attr* Path diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index f4bff80c4c..11d81d4778 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -345,7 +345,7 @@ impl ast::Literal { impl ast::FloatLiteral { pub fn suffix(&self) -> Option { - ast::FloatNumber::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) + ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) } } diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index b294849937..7c82372aa2 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1090,9 +1090,10 @@ pub struct FloatLiteral { pub(crate) syntax: SyntaxNode, } impl FloatLiteral { - pub fn float_number_token(&self) -> Option { - support::token(&self.syntax, T![float_number]) + pub fn float_number_part_token(&self) -> Option { + support::token(&self.syntax, T![float_number_part]) } + pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index a3209c5abd..1e1a55e326 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -112,16 +112,16 @@ impl AstToken for IntNumber { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatNumber { +pub struct FloatNumberPart { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for FloatNumber { +impl std::fmt::Display for FloatNumberPart { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for FloatNumber { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER } +impl AstToken for FloatNumberPart { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } fn cast(syntax: SyntaxToken) -> Option { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index f2153ca921..5b19b5ed2c 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -555,7 +555,9 @@ impl ast::FieldExpr { self.syntax .children_with_tokens() // FIXME: Accepting floats here to reject them in validation later - .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER) + .find(|c| { + c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART + }) .as_ref() .and_then(SyntaxElement::as_token) .cloned() diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 3063396b44..5f2e7231d9 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -321,7 +321,7 @@ impl ast::IntNumber { } } -impl ast::FloatNumber { +impl ast::FloatNumberPart { pub fn suffix(&self) -> Option<&str> { let text = self.text(); let mut indices = text.char_indices(); diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 964bd6c8b3..0a0632da74 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,7 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], + literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index e18f575e37..eb6a5f63ea 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -462,7 +462,7 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; - [float_number] => { $crate::SyntaxKind::FLOAT_NUMBER }; + [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; } pub use T; }; @@ -586,7 +586,7 @@ impl Field { fn lower(grammar: &Grammar) -> AstSrc { let mut res = AstSrc { - tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident" + tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" .split_ascii_whitespace() .map(|it| it.to_string()) .collect::>(), From 2a755495d0ccec02d6ed474d7022a92a392593cb Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 4 May 2022 16:58:02 +0200 Subject: [PATCH 4/9] Add tests --- crates/hir-ty/src/tests/simple.rs | 11 +++++++ crates/parser/src/grammar/expressions.rs | 2 ++ .../parser/inline/ok/0011_field_expr.rast | 33 +++++++++++++++++++ .../parser/inline/ok/0011_field_expr.rs | 2 ++ 4 files changed, 48 insertions(+) diff --git a/crates/hir-ty/src/tests/simple.rs b/crates/hir-ty/src/tests/simple.rs index d4d61c2167..4b8a7e782d 100644 --- a/crates/hir-ty/src/tests/simple.rs +++ b/crates/hir-ty/src/tests/simple.rs @@ -2733,3 +2733,14 @@ fn f() { "#, ); } + +#[test] +fn nested_tuple_index() { + check_no_mismatches( + r#" +fn main() { + let fld: i32 = ((0,),).0.0; +} +"#, + ); +} diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 85f53dd237..4189ec41b3 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -469,6 +469,8 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo; // x.0.bar; +// x.0. bar; +// x.0.1; // x.0(); // } fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast index 8498724b9e..a1efb3a9fb 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast @@ -40,6 +40,39 @@ SOURCE_FILE IDENT "bar" SEMICOLON ";" WHITESPACE "\n " + EXPR_STMT + FIELD_EXPR + FIELD_EXPR + PATH_EXPR + PATH + PATH_SEGMENT + NAME_REF + IDENT "x" + DOT "." + NAME_REF + FLOAT_NUMBER_PART "0" + DOT "." + WHITESPACE " " + NAME_REF + IDENT "bar" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + FIELD_EXPR + FIELD_EXPR + PATH_EXPR + PATH + PATH_SEGMENT + NAME_REF + IDENT "x" + DOT "." + NAME_REF + FLOAT_NUMBER_PART "0" + DOT "." + NAME_REF + FLOAT_NUMBER_PART "1" + SEMICOLON ";" + WHITESPACE "\n " EXPR_STMT CALL_EXPR FIELD_EXPR diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs index b8da2ddc30..551b1ecaf0 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs @@ -1,5 +1,7 @@ fn foo() { x.foo; x.0.bar; + x.0. bar; + x.0.1; x.0(); } From 90bd99f1bbf08d2558d32b6ede358a1f97302ee3 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 4 May 2022 17:10:53 +0200 Subject: [PATCH 5/9] Add completion test --- crates/ide-completion/src/completions/dot.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/ide-completion/src/completions/dot.rs b/crates/ide-completion/src/completions/dot.rs index 01a5f856c1..c660e8f8fa 100644 --- a/crates/ide-completion/src/completions/dot.rs +++ b/crates/ide-completion/src/completions/dot.rs @@ -785,4 +785,24 @@ fn main() { ", ) } + + #[test] + fn tuple_index_completion() { + check( + r#" +struct I; +impl I { + fn i_method(&self) {} +} +struct S((), I); + +fn f(s: S) { + s.1.$0 +} +"#, + expect![[r#" + me i_method() fn(&self) + "#]], + ); + } } From 34dc8e93835d80ed4825bfad6e4979e6c4a4076a Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Wed, 4 May 2022 19:25:32 +0200 Subject: [PATCH 6/9] Wrap floats in token trees in `FLOAT_LITERAL` node --- .../ide/src/syntax_highlighting/highlight.rs | 3 +++ .../test_data/highlight_strings.html | 14 +++++------ crates/parser/src/grammar/expressions.rs | 2 +- crates/parser/src/grammar/expressions/atom.rs | 24 +++++++++++-------- crates/parser/src/grammar/items.rs | 5 +++- .../parser/err/0023_mismatched_paren.rast | 7 +++--- 6 files changed, 33 insertions(+), 22 deletions(-) diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index f061e8637e..2e3271dc31 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -31,6 +31,9 @@ pub(super) fn token(sema: &Semantics, token: SyntaxToken) -> Optio SymbolKind::Field.into() } INT_NUMBER | FLOAT_NUMBER_PART => HlTag::NumericLiteral.into(), + DOT if token.parent().map(|n| n.kind()) == Some(FLOAT_LITERAL) => { + HlTag::NumericLiteral.into() + } BYTE => HlTag::ByteLiteral.into(), CHAR => HlTag::CharLiteral.into(), IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index 329184730e..d7d6c79e3d 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd println!("Hello {:05}!", 5); println!("Hello {:05}!", -5); println!("{:#010x}!", 27); - println!("Hello {0} is {1:.5}", "x", 0.01); - println!("Hello {1} is {2:.0$}", 5, "x", 0.01); - println!("Hello {0} is {2:.1$}", "x", 5, 0.01); - println!("Hello {} is {:.*}", "x", 5, 0.01); - println!("Hello {} is {2:.*}", "x", 5, 0.01); - println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); - println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); + println!("Hello {0} is {1:.5}", "x", 0.01); + println!("Hello {1} is {2:.0$}", 5, "x", 0.01); + println!("Hello {0} is {2:.1$}", "x", 5, 0.01); + println!("Hello {} is {:.*}", "x", 5, 0.01); + println!("Hello {} is {2:.*}", "x", 5, 0.01); + println!("Hello {} is {number:.prec$}", "x", prec = 5, number = 0.01); + println!("{}, `{name:.*}` has 3 fractional digits", "Hello", 3, name=1234.56); println!("{}, `{name:.*}` has 3 characters", "Hello", 3, name="1234.56"); println!("{}, `{name:>8.*}` has 3 right-aligned characters", "Hello", 3, name="1234.56"); println!("Hello {{}}"); diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 4189ec41b3..cb384cce81 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ mod atom; use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{literal, LITERAL_FIRST}; +pub(super) use self::atom::{float_literal, literal, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 07b0a2aee5..d7a0691717 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -30,16 +30,7 @@ pub(crate) fn literal(p: &mut Parser) -> Option { } let m = p.start(); if p.at(FLOAT_NUMBER_PART) { - // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT` - let f = p.start(); - p.bump(FLOAT_NUMBER_PART); - if p.at(DOT) { - p.bump(DOT); - if p.at(FLOAT_NUMBER_PART) { - p.bump(FLOAT_NUMBER_PART); - } - } - f.complete(p, FLOAT_LITERAL); + float_literal(p); } else { // Everything else is just one token. p.bump_any(); @@ -47,6 +38,19 @@ pub(crate) fn literal(p: &mut Parser) -> Option { Some(m.complete(p, LITERAL)) } +pub(crate) fn float_literal(p: &mut Parser) { + // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT` + let f = p.start(); + p.bump(FLOAT_NUMBER_PART); + if p.at(DOT) { + p.bump(DOT); + if p.at(FLOAT_NUMBER_PART) { + p.bump(FLOAT_NUMBER_PART); + } + } + f.complete(p, FLOAT_LITERAL); +} + // E.g. for after the break in `if break {}`, this should not match pub(super) const ATOM_EXPR_FIRST: TokenSet = LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[ diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs index 7bfd9ef8c8..3d2f788e9a 100644 --- a/crates/parser/src/grammar/items.rs +++ b/crates/parser/src/grammar/items.rs @@ -9,7 +9,7 @@ pub(crate) use self::{ traits::assoc_item_list, use_item::use_tree_list, }; -use super::*; +use super::{expressions::float_literal, *}; // test mod_contents // fn foo() {} @@ -457,6 +457,9 @@ pub(crate) fn token_tree(p: &mut Parser) { return; } T![')'] | T![']'] => p.err_and_bump("unmatched brace"), + FLOAT_NUMBER_PART => { + float_literal(p); + } _ => p.bump_any(), } } diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index f57e58a4b7..a13dcd77f5 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,9 +32,10 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_NUMBER_PART "2" - DOT "." - FLOAT_NUMBER_PART "0" + FLOAT_LITERAL + FLOAT_NUMBER_PART "2" + DOT "." + FLOAT_NUMBER_PART "0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " From 2fe38d3b63cf4b6f2278a656b7ffed40a524f0ff Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 5 May 2022 16:06:44 +0200 Subject: [PATCH 7/9] Indicate the number of float tokens in the first token --- crates/parser/src/grammar.rs | 11 +++- crates/parser/src/grammar/expressions.rs | 7 ++- crates/parser/src/grammar/expressions/atom.rs | 36 ++++++++++--- crates/parser/src/grammar/items.rs | 2 +- crates/parser/src/lexed_str.rs | 13 +++-- crates/parser/src/syntax_kind/generated.rs | 8 ++- .../test_data/lexer/err/empty_exponent.rast | 28 +++++----- crates/parser/test_data/lexer/ok/numbers.rast | 16 +++--- .../inline/ok/0107_method_call_expr.rast | 26 ++++++++++ .../parser/inline/ok/0107_method_call_expr.rs | 3 ++ .../parser/inline/ok/0201_float_literal.rast | 51 +++++++++++++++++++ .../parser/inline/ok/0201_float_literal.rs | 7 +++ crates/syntax/rust.ungram | 4 +- crates/syntax/src/ast/generated/nodes.rs | 11 +++- crates/syntax/src/tests/ast_src.rs | 12 ++++- crates/syntax/src/tests/sourcegen_ast.rs | 3 ++ 16 files changed, 194 insertions(+), 44 deletions(-) create mode 100644 crates/parser/test_data/parser/inline/ok/0201_float_literal.rast create mode 100644 crates/parser/test_data/parser/inline/ok/0201_float_literal.rs diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 45d9b2e4e0..92f3302538 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -39,6 +39,7 @@ mod generic_params; mod types; use crate::{ + grammar::expressions::FLOAT_LITERAL_FIRST, parser::{CompletedMarker, Marker, Parser}, SyntaxKind::{self, *}, TokenSet, T, @@ -318,9 +319,15 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART)); + assert!( + p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) + ); let m = p.start(); - p.bump_any(); + if p.at_ts(FLOAT_LITERAL_FIRST) { + p.bump_remap(FLOAT_NUMBER_PART); + } else { + p.bump_any(); + } m.complete(p, NAME_REF); } diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index cb384cce81..15dba362a9 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ mod atom; use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{float_literal, literal, LITERAL_FIRST}; +pub(super) use self::atom::{float_literal, literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { @@ -452,6 +452,9 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo(); // y.bar::(1, 2,); +// +// 0e0.sin(); +// 0e0f32.sin(); // } fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); @@ -477,7 +480,7 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) { + if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { name_ref_or_index(p); } else { p.error("expected field name or number"); diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index d7a0691717..c191b5593d 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,19 +17,24 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER_PART, + FLOAT_NUMBER_START_0, + FLOAT_NUMBER_START_1, + FLOAT_NUMBER_START_2, BYTE, CHAR, STRING, BYTE_STRING, ]); +pub(crate) const FLOAT_LITERAL_FIRST: TokenSet = + TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]); + pub(crate) fn literal(p: &mut Parser) -> Option { if !p.at_ts(LITERAL_FIRST) { return None; } let m = p.start(); - if p.at(FLOAT_NUMBER_PART) { + if p.at_ts(FLOAT_LITERAL_FIRST) { float_literal(p); } else { // Everything else is just one token. @@ -38,15 +43,30 @@ pub(crate) fn literal(p: &mut Parser) -> Option { Some(m.complete(p, LITERAL)) } +// test float_literal +// fn f() { +// 0.0; +// 1.; +// 0e0; +// 0e0f32; +// 1.23f64; +// } pub(crate) fn float_literal(p: &mut Parser) { - // Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT` + // Floats can be up to 3 tokens. The first token indicates how many there are. + // We remap the first token to `FLOAT_NUMBER_PART` so that no subsequent code has to deal with + // this awful, awful hack. let f = p.start(); - p.bump(FLOAT_NUMBER_PART); - if p.at(DOT) { + if p.at(FLOAT_NUMBER_START_0) { + p.bump_remap(FLOAT_NUMBER_PART); + } else if p.at(FLOAT_NUMBER_START_1) { + p.bump_remap(FLOAT_NUMBER_PART); p.bump(DOT); - if p.at(FLOAT_NUMBER_PART) { - p.bump(FLOAT_NUMBER_PART); - } + } else if p.at(FLOAT_NUMBER_START_2) { + p.bump_remap(FLOAT_NUMBER_PART); + p.bump(DOT); + p.bump(FLOAT_NUMBER_PART); + } else { + unreachable!(); } f.complete(p, FLOAT_LITERAL); } diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs index 3d2f788e9a..6516603402 100644 --- a/crates/parser/src/grammar/items.rs +++ b/crates/parser/src/grammar/items.rs @@ -457,7 +457,7 @@ pub(crate) fn token_tree(p: &mut Parser) { return; } T![')'] | T![']'] => p.err_and_bump("unmatched brace"), - FLOAT_NUMBER_PART => { + FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { float_literal(p); } _ => p.bump_any(), diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index fae5d8884b..9887960871 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -240,11 +240,16 @@ impl<'a> Converter<'a> { // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` // is lexed as a float, we split floats that contain a `.` into 3 tokens. + // To ensure that later stages can always reconstruct the token correctly, the first + // token in the sequence indicates the number of following tokens that are part of + // the float literal. if let Some((before, after)) = token_text.split_once('.') { let err = if err.is_empty() { None } else { Some(err) }; - if !before.is_empty() { - self.push(FLOAT_NUMBER_PART, before.len(), None); - } + + assert!(!before.is_empty()); + let tok = + if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 }; + self.push(tok, before.len(), None); self.push(DOT, 1, None); if !after.is_empty() { self.push(FLOAT_NUMBER_PART, after.len(), err); @@ -252,7 +257,7 @@ impl<'a> Converter<'a> { return; } - FLOAT_NUMBER_PART + FLOAT_NUMBER_START_0 } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 31fe1b2aba..5ddcd6ad6a 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,6 +110,9 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, + FLOAT_NUMBER_START_0, + FLOAT_NUMBER_START_1, + FLOAT_NUMBER_START_2, FLOAT_NUMBER_PART, CHAR, BYTE, @@ -287,7 +290,8 @@ impl SyntaxKind { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 + | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -387,5 +391,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; [float_number_start_0] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_0 } ; [float_number_start_1] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_1 } ; [float_number_start_2] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_2 } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index d0a268b8c1..73de4cac24 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER_PART "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_PART "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,35 +30,35 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_PART "42" +FLOAT_NUMBER_START_2 "42" DOT "." FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 689422933c..428bdf8a1f 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,7 +4,7 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER_PART "0" +FLOAT_NUMBER_START_1 "0" DOT "." WHITESPACE " " INT_NUMBER "0z" @@ -21,13 +21,13 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER_PART "0" +FLOAT_NUMBER_START_2 "0" DOT "." FLOAT_NUMBER_PART "1279" WHITESPACE " " -FLOAT_NUMBER_PART "0e1279" +FLOAT_NUMBER_START_0 "0e1279" WHITESPACE " " -FLOAT_NUMBER_PART "0E1279" +FLOAT_NUMBER_START_0 "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -40,7 +40,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER_PART "0e+1" +FLOAT_NUMBER_START_0 "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -48,17 +48,17 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER_PART "0" +FLOAT_NUMBER_START_2 "0" DOT "." FLOAT_NUMBER_PART "0E-2" WHITESPACE "\n" -FLOAT_NUMBER_PART "0___0" +FLOAT_NUMBER_START_2 "0___0" DOT "." FLOAT_NUMBER_PART "10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER_PART "92" +FLOAT_NUMBER_START_2 "92" DOT "." FLOAT_NUMBER_PART "0f32" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast index dcbcfe1231..0a14e20da8 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast @@ -58,6 +58,32 @@ SOURCE_FILE COMMA "," R_PAREN ")" SEMICOLON ";" + WHITESPACE "\n\n " + EXPR_STMT + METHOD_CALL_EXPR + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0" + DOT "." + NAME_REF + IDENT "sin" + ARG_LIST + L_PAREN "(" + R_PAREN ")" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + METHOD_CALL_EXPR + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0f32" + DOT "." + NAME_REF + IDENT "sin" + ARG_LIST + L_PAREN "(" + R_PAREN ")" + SEMICOLON ";" WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs index 1a3aa35ae8..3e5d464e23 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs @@ -1,4 +1,7 @@ fn foo() { x.foo(); y.bar::(1, 2,); + + 0e0.sin(); + 0e0f32.sin(); } diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast new file mode 100644 index 0000000000..a5cd2dffa4 --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast @@ -0,0 +1,51 @@ +SOURCE_FILE + FN + FN_KW "fn" + WHITESPACE " " + NAME + IDENT "f" + PARAM_LIST + L_PAREN "(" + R_PAREN ")" + WHITESPACE " " + BLOCK_EXPR + STMT_LIST + L_CURLY "{" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0" + DOT "." + FLOAT_NUMBER_PART "0" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "1" + DOT "." + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "0e0f32" + SEMICOLON ";" + WHITESPACE "\n " + EXPR_STMT + LITERAL + FLOAT_LITERAL + FLOAT_NUMBER_PART "1" + DOT "." + FLOAT_NUMBER_PART "23f64" + SEMICOLON ";" + WHITESPACE "\n" + R_CURLY "}" + WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs new file mode 100644 index 0000000000..0d51ec1252 --- /dev/null +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs @@ -0,0 +1,7 @@ +fn f() { + 0.0; + 1.; + 0e0; + 0e0f32; + 1.23f64; +} diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 8e4f07d3b5..85270dee62 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -373,7 +373,9 @@ Literal = ) FloatLiteral = - 'float_number_part' + 'float_number_start_0'? + 'float_number_start_1'? + 'float_number_start_2'? '.'? 'float_number_part'? diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index 7c82372aa2..9d5af8e63c 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1090,10 +1090,19 @@ pub struct FloatLiteral { pub(crate) syntax: SyntaxNode, } impl FloatLiteral { + pub fn float_number_start_0_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_0]) + } + pub fn float_number_start_1_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_1]) + } + pub fn float_number_start_2_token(&self) -> Option { + support::token(&self.syntax, T![float_number_start_2]) + } + pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } pub fn float_number_part_token(&self) -> Option { support::token(&self.syntax, T![float_number_part]) } - pub fn dot_token(&self) -> Option { support::token(&self.syntax, T![.]) } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 0a0632da74..f5a78e4119 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,7 +71,17 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &["INT_NUMBER", "FLOAT_NUMBER_PART", "CHAR", "BYTE", "STRING", "BYTE_STRING"], + literals: &[ + "INT_NUMBER", + "FLOAT_NUMBER_START_0", + "FLOAT_NUMBER_START_1", + "FLOAT_NUMBER_START_2", + "FLOAT_NUMBER_PART", + "CHAR", + "BYTE", + "STRING", + "BYTE_STRING", + ], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index eb6a5f63ea..33bf2c0cba 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -463,6 +463,9 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; + [float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 }; + [float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 }; + [float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 }; } pub use T; }; From 37443eb9a159af17d95680c13e693d46344e4bc6 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 5 May 2022 16:26:16 +0200 Subject: [PATCH 8/9] Maybe everything else *should* have to deal with it --- .../ide/src/syntax_highlighting/highlight.rs | 9 ++++++-- crates/mbe/src/syntax_bridge.rs | 22 ++++++++----------- crates/parser/src/grammar/expressions.rs | 2 +- crates/parser/src/grammar/expressions/atom.rs | 8 +++---- crates/parser/src/grammar/items.rs | 5 +---- .../parser/err/0023_mismatched_paren.rast | 7 +++--- .../parser/inline/ok/0085_expr_literals.rast | 2 +- .../inline/ok/0107_method_call_expr.rast | 4 ++-- .../parser/inline/ok/0201_float_literal.rast | 10 ++++----- .../test_data/parser/ok/0056_neq_in_type.rast | 4 ++-- 10 files changed, 34 insertions(+), 39 deletions(-) diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index 2e3271dc31..bafbf6d9cc 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -30,8 +30,13 @@ pub(super) fn token(sema: &Semantics, token: SyntaxToken) -> Optio INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { SymbolKind::Field.into() } - INT_NUMBER | FLOAT_NUMBER_PART => HlTag::NumericLiteral.into(), - DOT if token.parent().map(|n| n.kind()) == Some(FLOAT_LITERAL) => { + INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 + | FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(), + DOT if matches!( + token.prev_token().map(|n| n.kind()), + Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2) + ) => + { HlTag::NumericLiteral.into() } BYTE => HlTag::ByteLiteral.into(), diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 8b8577986d..79fd1250e0 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -260,23 +260,19 @@ fn convert_tokens(conv: &mut C) -> tt::Subtree { IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), - FLOAT_NUMBER_PART => { + FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { // Reassemble a split-up float token. let mut range = range; let mut text = token.to_text(conv).to_string(); - if let Some(dot) = conv.peek() { - if dot.kind(conv) == DOT { - let (_, dot_range) = conv.bump().unwrap(); - text += &*dot.to_text(conv); - range = TextRange::new(range.start(), dot_range.end()); + if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 { + let (dot, dot_range) = conv.bump().unwrap(); + text += &*dot.to_text(conv); + range = TextRange::new(range.start(), dot_range.end()); - if let Some(tail) = conv.peek() { - if tail.kind(conv) == FLOAT_NUMBER_PART { - let (_, tail_range) = conv.bump().unwrap(); - text += &*tail.to_text(conv); - range = TextRange::new(range.start(), tail_range.end()); - } - } + if kind == FLOAT_NUMBER_START_2 { + let (tail, tail_range) = conv.bump().unwrap(); + text += &*tail.to_text(conv); + range = TextRange::new(range.start(), tail_range.end()); } } diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 15dba362a9..290083b343 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ mod atom; use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{float_literal, literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; +pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index c191b5593d..5ab148ff28 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -53,16 +53,14 @@ pub(crate) fn literal(p: &mut Parser) -> Option { // } pub(crate) fn float_literal(p: &mut Parser) { // Floats can be up to 3 tokens. The first token indicates how many there are. - // We remap the first token to `FLOAT_NUMBER_PART` so that no subsequent code has to deal with - // this awful, awful hack. let f = p.start(); if p.at(FLOAT_NUMBER_START_0) { - p.bump_remap(FLOAT_NUMBER_PART); + p.bump(FLOAT_NUMBER_START_0); } else if p.at(FLOAT_NUMBER_START_1) { - p.bump_remap(FLOAT_NUMBER_PART); + p.bump(FLOAT_NUMBER_START_1); p.bump(DOT); } else if p.at(FLOAT_NUMBER_START_2) { - p.bump_remap(FLOAT_NUMBER_PART); + p.bump(FLOAT_NUMBER_START_2); p.bump(DOT); p.bump(FLOAT_NUMBER_PART); } else { diff --git a/crates/parser/src/grammar/items.rs b/crates/parser/src/grammar/items.rs index 6516603402..7bfd9ef8c8 100644 --- a/crates/parser/src/grammar/items.rs +++ b/crates/parser/src/grammar/items.rs @@ -9,7 +9,7 @@ pub(crate) use self::{ traits::assoc_item_list, use_item::use_tree_list, }; -use super::{expressions::float_literal, *}; +use super::*; // test mod_contents // fn foo() {} @@ -457,9 +457,6 @@ pub(crate) fn token_tree(p: &mut Parser) { return; } T![')'] | T![']'] => p.err_and_bump("unmatched brace"), - FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { - float_literal(p); - } _ => p.bump_any(), } } diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index a13dcd77f5..70cd030da1 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,10 +32,9 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_LITERAL - FLOAT_NUMBER_PART "2" - DOT "." - FLOAT_NUMBER_PART "0" + FLOAT_NUMBER_START_2 "2" + DOT "." + FLOAT_NUMBER_PART "0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index 367aff324a..b3236976b9 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -58,7 +58,7 @@ SOURCE_FILE WHITESPACE " " LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "2" + FLOAT_NUMBER_START_2 "2" DOT "." FLOAT_NUMBER_PART "0" SEMICOLON ";" diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast index 0a14e20da8..69f1055b7e 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast @@ -63,7 +63,7 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "0e0" + FLOAT_NUMBER_START_0 "0e0" DOT "." NAME_REF IDENT "sin" @@ -76,7 +76,7 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "0e0f32" + FLOAT_NUMBER_START_0 "0e0f32" DOT "." NAME_REF IDENT "sin" diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast index a5cd2dffa4..df4fb6eb41 100644 --- a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast +++ b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast @@ -15,7 +15,7 @@ SOURCE_FILE EXPR_STMT LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "0" + FLOAT_NUMBER_START_2 "0" DOT "." FLOAT_NUMBER_PART "0" SEMICOLON ";" @@ -23,26 +23,26 @@ SOURCE_FILE EXPR_STMT LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "1" + FLOAT_NUMBER_START_1 "1" DOT "." SEMICOLON ";" WHITESPACE "\n " EXPR_STMT LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "0e0" + FLOAT_NUMBER_START_0 "0e0" SEMICOLON ";" WHITESPACE "\n " EXPR_STMT LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "0e0f32" + FLOAT_NUMBER_START_0 "0e0f32" SEMICOLON ";" WHITESPACE "\n " EXPR_STMT LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "1" + FLOAT_NUMBER_START_2 "1" DOT "." FLOAT_NUMBER_PART "23f64" SEMICOLON ";" diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index 5fd9271c62..b2f66e2f1e 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -20,7 +20,7 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "1" + FLOAT_NUMBER_START_2 "1" DOT "." FLOAT_NUMBER_PART "0f32" DOT "." @@ -44,7 +44,7 @@ SOURCE_FILE METHOD_CALL_EXPR LITERAL FLOAT_LITERAL - FLOAT_NUMBER_PART "1" + FLOAT_NUMBER_START_2 "1" DOT "." FLOAT_NUMBER_PART "0f32" DOT "." From d974a0b8894209a041fb06c3c8713d7e8b91cbb7 Mon Sep 17 00:00:00 2001 From: Jonas Schievink Date: Thu, 5 May 2022 16:35:07 +0200 Subject: [PATCH 9/9] Fix rebase fallout --- crates/hir-expand/src/builtin_fn_macro.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 45eb660a6c..687c549748 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -440,7 +440,7 @@ fn concat_bytes_expand( .into_iter() .for_each(|x| bytes.push(x.to_string())); } - ast::LiteralKind::Byte => { + ast::LiteralKind::Byte(_) => { bytes.push(lit.to_string()); } _ => { @@ -477,7 +477,7 @@ fn concat_bytes_expand_subtree( tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { let lit = ast::make::literal(&lit.to_string()); match lit.kind() { - ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte => { + ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => { bytes.push(lit.to_string()); } _ => {