⬆️ rust-analyzer

crates/parser/src/event.rs

@@ -74,7 +74,13 @@ pub(crate) enum Event {
         kind: SyntaxKind,
         n_raw_tokens: u8,
     },
-
+    /// When we parse `foo.0.0` or `foo. 0. 0` the lexer will hand us a float literal
+    /// instead of an integer literal followed by a dot as the lexer has no contextual knowledge.
+    /// This event instructs whatever consumes the events to split the float literal into
+    /// the corresponding parts.
+    FloatSplitHack {
+        ends_in_dot: bool,
+    },
     Error {
         msg: String,
     },
@@ -125,6 +131,11 @@ pub(super) fn process(mut events: Vec<Event>) -> Output {
             Event::Token { kind, n_raw_tokens } => {
                 res.token(kind, n_raw_tokens);
             }
+            Event::FloatSplitHack { ends_in_dot } => {
+                res.float_split_hack(ends_in_dot);
+                let ev = mem::replace(&mut events[i + 1], Event::tombstone());
+                assert!(matches!(ev, Event::Finish), "{ev:?}");
+            }
             Event::Error { msg } => res.error(msg),
         }
     }

crates/parser/src/grammar/expressions.rs

@@ -379,7 +379,7 @@ fn postfix_expr(
             // }
             T!['('] if allow_calls => call_expr(p, lhs),
             T!['['] if allow_calls => index_expr(p, lhs),
-            T![.] => match postfix_dot_expr(p, lhs) {
+            T![.] => match postfix_dot_expr::<false>(p, lhs) {
                 Ok(it) => it,
                 Err(it) => {
                     lhs = it;
@@ -393,35 +393,44 @@ fn postfix_expr(
         block_like = BlockLike::NotBlock;
     }
     return (lhs, block_like);
+}
 
-    fn postfix_dot_expr(
-        p: &mut Parser<'_>,
-        lhs: CompletedMarker,
-    ) -> Result<CompletedMarker, CompletedMarker> {
+fn postfix_dot_expr<const FLOAT_RECOVERY: bool>(
+    p: &mut Parser<'_>,
+    lhs: CompletedMarker,
+) -> Result<CompletedMarker, CompletedMarker> {
+    if !FLOAT_RECOVERY {
         assert!(p.at(T![.]));
-        if p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])) {
-            return Ok(method_call_expr(p, lhs));
-        }
-
-        // test await_expr
-        // fn foo() {
-        //     x.await;
-        //     x.0.await;
-        //     x.0().await?.hello();
-        // }
-        if p.nth(1) == T![await] {
-            let m = lhs.precede(p);
-            p.bump(T![.]);
-            p.bump(T![await]);
-            return Ok(m.complete(p, AWAIT_EXPR));
-        }
-
-        if p.at(T![..=]) || p.at(T![..]) {
-            return Err(lhs);
-        }
-
-        Ok(field_expr(p, lhs))
     }
+    let nth1 = if FLOAT_RECOVERY { 0 } else { 1 };
+    let nth2 = if FLOAT_RECOVERY { 1 } else { 2 };
+
+    if p.nth(nth1) == IDENT && (p.nth(nth2) == T!['('] || p.nth_at(nth2, T![::])) {
+        return Ok(method_call_expr::<FLOAT_RECOVERY>(p, lhs));
+    }
+
+    // test await_expr
+    // fn foo() {
+    //     x.await;
+    //     x.0.await;
+    //     x.0().await?.hello();
+    //     x.0.0.await;
+    //     x.0. await;
+    // }
+    if p.nth(nth1) == T![await] {
+        let m = lhs.precede(p);
+        if !FLOAT_RECOVERY {
+            p.bump(T![.]);
+        }
+        p.bump(T![await]);
+        return Ok(m.complete(p, AWAIT_EXPR));
+    }
+
+    if p.at(T![..=]) || p.at(T![..]) {
+        return Err(lhs);
+    }
+
+    field_expr::<FLOAT_RECOVERY>(p, lhs)
 }
 
 // test call_expr
@@ -455,11 +464,22 @@ fn index_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
 // fn foo() {
 //     x.foo();
 //     y.bar::<T>(1, 2,);
+//     x.0.0.call();
+//     x.0. call();
 // }
-fn method_call_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
-    assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
+fn method_call_expr<const FLOAT_RECOVERY: bool>(
+    p: &mut Parser<'_>,
+    lhs: CompletedMarker,
+) -> CompletedMarker {
+    if FLOAT_RECOVERY {
+        assert!(p.nth(0) == IDENT && (p.nth(1) == T!['('] || p.nth_at(1, T![::])));
+    } else {
+        assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::])));
+    }
     let m = lhs.precede(p);
-    p.bump_any();
+    if !FLOAT_RECOVERY {
+        p.bump(T![.]);
+    }
     name_ref(p);
     generic_args::opt_generic_arg_list(p, true);
     if p.at(T!['(']) {
@@ -472,21 +492,35 @@ fn method_call_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker
 // fn foo() {
 //     x.foo;
 //     x.0.bar;
+//     x.0.1;
+//     x.0. bar;
 //     x.0();
 // }
-fn field_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker {
-    assert!(p.at(T![.]));
+fn field_expr<const FLOAT_RECOVERY: bool>(
+    p: &mut Parser<'_>,
+    lhs: CompletedMarker,
+) -> Result<CompletedMarker, CompletedMarker> {
+    if !FLOAT_RECOVERY {
+        assert!(p.at(T![.]));
+    }
     let m = lhs.precede(p);
-    p.bump(T![.]);
+    if !FLOAT_RECOVERY {
+        p.bump(T![.]);
+    }
     if p.at(IDENT) || p.at(INT_NUMBER) {
         name_ref_or_index(p);
     } else if p.at(FLOAT_NUMBER) {
-        // FIXME: How to recover and instead parse INT + T![.]?
-        p.bump_any();
+        return match p.split_float(m) {
+            (true, m) => {
+                let lhs = m.complete(p, FIELD_EXPR);
+                postfix_dot_expr::<true>(p, lhs)
+            }
+            (false, m) => Ok(m.complete(p, FIELD_EXPR)),
+        };
     } else {
         p.error("expected field name or number");
     }
-    m.complete(p, FIELD_EXPR)
+    Ok(m.complete(p, FIELD_EXPR))
 }
 
 // test try_expr

crates/parser/src/grammar/expressions/atom.rs

@@ -152,7 +152,7 @@ pub(super) fn atom_expr(
             m.complete(p, BLOCK_EXPR)
         }
 
-        T![static] | T![async] | T![move] | T![|] => closure_expr(p),
+        T![const] | T![static] | T![async] | T![move] | T![|] => closure_expr(p),
         T![for] if la == T![<] => closure_expr(p),
         T![for] => for_expr(p, None),
 
@@ -255,7 +255,7 @@ fn array_expr(p: &mut Parser<'_>) -> CompletedMarker {
 // }
 fn closure_expr(p: &mut Parser<'_>) -> CompletedMarker {
     assert!(match p.current() {
-        T![static] | T![async] | T![move] | T![|] => true,
+        T![const] | T![static] | T![async] | T![move] | T![|] => true,
         T![for] => p.nth(1) == T![<],
         _ => false,
     });
@@ -265,7 +265,9 @@ fn closure_expr(p: &mut Parser<'_>) -> CompletedMarker {
     if p.at(T![for]) {
         types::for_binder(p);
     }
-
+    // test const_closure
+    // fn main() { let cl = const || _ = 0; }
+    p.eat(T![const]);
     p.eat(T![static]);
     p.eat(T![async]);
     p.eat(T![move]);

crates/parser/src/lexed_str.rs

@@ -82,6 +82,7 @@ impl<'a> LexedStr<'a> {
     pub fn text(&self, i: usize) -> &str {
         self.range_text(i..i + 1)
     }
+
     pub fn range_text(&self, r: ops::Range<usize>) -> &str {
         assert!(r.start < r.end && r.end <= self.len());
         let lo = self.start[r.start] as usize;
@@ -216,6 +217,10 @@ impl<'a> Converter<'a> {
                 rustc_lexer::TokenKind::Caret => T![^],
                 rustc_lexer::TokenKind::Percent => T![%],
                 rustc_lexer::TokenKind::Unknown => ERROR,
+                rustc_lexer::TokenKind::UnknownPrefix => {
+                    err = "unknown literal prefix";
+                    IDENT
+                }
             }
         };
 

crates/parser/src/lib.rs

@@ -102,10 +102,14 @@ impl TopEntryPoint {
                 match step {
                     Step::Enter { .. } => depth += 1,
                     Step::Exit => depth -= 1,
+                    Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
+                        depth -= 1 + !has_pseudo_dot as usize
+                    }
                     Step::Token { .. } | Step::Error { .. } => (),
                 }
             }
             assert!(!first, "no tree at all");
+            assert_eq!(depth, 0, "unbalanced tree");
         }
 
         res

crates/parser/src/output.rs

@@ -25,53 +25,88 @@ pub struct Output {
 #[derive(Debug)]
 pub enum Step<'a> {
     Token { kind: SyntaxKind, n_input_tokens: u8 },
+    FloatSplit { ends_in_dot: bool },
     Enter { kind: SyntaxKind },
     Exit,
     Error { msg: &'a str },
 }
 
 impl Output {
+    const EVENT_MASK: u32 = 0b1;
+    const TAG_MASK: u32 = 0x0000_00F0;
+    const N_INPUT_TOKEN_MASK: u32 = 0x0000_FF00;
+    const KIND_MASK: u32 = 0xFFFF_0000;
+
+    const ERROR_SHIFT: u32 = Self::EVENT_MASK.trailing_ones();
+    const TAG_SHIFT: u32 = Self::TAG_MASK.trailing_zeros();
+    const N_INPUT_TOKEN_SHIFT: u32 = Self::N_INPUT_TOKEN_MASK.trailing_zeros();
+    const KIND_SHIFT: u32 = Self::KIND_MASK.trailing_zeros();
+
+    const TOKEN_EVENT: u8 = 0;
+    const ENTER_EVENT: u8 = 1;
+    const EXIT_EVENT: u8 = 2;
+    const SPLIT_EVENT: u8 = 3;
+
     pub fn iter(&self) -> impl Iterator<Item = Step<'_>> {
         self.event.iter().map(|&event| {
-            if event & 0b1 == 0 {
-                return Step::Error { msg: self.error[(event as usize) >> 1].as_str() };
+            if event & Self::EVENT_MASK == 0 {
+                return Step::Error {
+                    msg: self.error[(event as usize) >> Self::ERROR_SHIFT].as_str(),
+                };
             }
-            let tag = ((event & 0x0000_00F0) >> 4) as u8;
+            let tag = ((event & Self::TAG_MASK) >> Self::TAG_SHIFT) as u8;
             match tag {
-                0 => {
-                    let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
-                    let n_input_tokens = ((event & 0x0000_FF00) >> 8) as u8;
+                Self::TOKEN_EVENT => {
+                    let kind: SyntaxKind =
+                        (((event & Self::KIND_MASK) >> Self::KIND_SHIFT) as u16).into();
+                    let n_input_tokens =
+                        ((event & Self::N_INPUT_TOKEN_MASK) >> Self::N_INPUT_TOKEN_SHIFT) as u8;
                     Step::Token { kind, n_input_tokens }
                 }
-                1 => {
-                    let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();
+                Self::ENTER_EVENT => {
+                    let kind: SyntaxKind =
+                        (((event & Self::KIND_MASK) >> Self::KIND_SHIFT) as u16).into();
                     Step::Enter { kind }
                 }
-                2 => Step::Exit,
+                Self::EXIT_EVENT => Step::Exit,
+                Self::SPLIT_EVENT => {
+                    Step::FloatSplit { ends_in_dot: event & Self::N_INPUT_TOKEN_MASK != 0 }
+                }
                 _ => unreachable!(),
             }
         })
     }
 
     pub(crate) fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {
-        let e = ((kind as u16 as u32) << 16) | ((n_tokens as u32) << 8) | 1;
+        let e = ((kind as u16 as u32) << Self::KIND_SHIFT)
+            | ((n_tokens as u32) << Self::N_INPUT_TOKEN_SHIFT)
+            | Self::EVENT_MASK;
         self.event.push(e)
     }
 
+    pub(crate) fn float_split_hack(&mut self, ends_in_dot: bool) {
+        let e = (Self::SPLIT_EVENT as u32) << Self::TAG_SHIFT
+            | ((ends_in_dot as u32) << Self::N_INPUT_TOKEN_SHIFT)
+            | Self::EVENT_MASK;
+        self.event.push(e);
+    }
+
     pub(crate) fn enter_node(&mut self, kind: SyntaxKind) {
-        let e = ((kind as u16 as u32) << 16) | (1 << 4) | 1;
+        let e = ((kind as u16 as u32) << Self::KIND_SHIFT)
+            | ((Self::ENTER_EVENT as u32) << Self::TAG_SHIFT)
+            | Self::EVENT_MASK;
         self.event.push(e)
     }
 
     pub(crate) fn leave_node(&mut self) {
-        let e = 2 << 4 | 1;
+        let e = (Self::EXIT_EVENT as u32) << Self::TAG_SHIFT | Self::EVENT_MASK;
         self.event.push(e)
     }
 
     pub(crate) fn error(&mut self, error: String) {
         let idx = self.error.len();
         self.error.push(error);
-        let e = (idx as u32) << 1;
+        let e = (idx as u32) << Self::ERROR_SHIFT;
         self.event.push(e);
     }
 }

crates/parser/src/parser.rs

@@ -181,6 +181,35 @@ impl<'t> Parser<'t> {
         self.do_bump(kind, 1);
     }
 
+    /// Advances the parser by one token
+    pub(crate) fn split_float(&mut self, mut marker: Marker) -> (bool, Marker) {
+        assert!(self.at(SyntaxKind::FLOAT_NUMBER));
+        // we have parse `<something>.`
+        // `<something>`.0.1
+        // here we need to insert an extra event
+        //
+        // `<something>`. 0. 1;
+        // here we need to change the follow up parse, the return value will cause us to emulate a dot
+        // the actual splitting happens later
+        let ends_in_dot = !self.inp.is_joint(self.pos);
+        if !ends_in_dot {
+            let new_marker = self.start();
+            let idx = marker.pos as usize;
+            match &mut self.events[idx] {
+                Event::Start { forward_parent, kind } => {
+                    *kind = SyntaxKind::FIELD_EXPR;
+                    *forward_parent = Some(new_marker.pos - marker.pos);
+                }
+                _ => unreachable!(),
+            }
+            marker.bomb.defuse();
+            marker = new_marker;
+        };
+        self.pos += 1 as usize;
+        self.push_event(Event::FloatSplitHack { ends_in_dot });
+        (ends_in_dot, marker)
+    }
+
     /// Advances the parser by one token, remapping its kind.
     /// This is useful to create contextual keywords from
     /// identifiers. For example, the lexer creates a `union`

crates/parser/src/shortcuts.rs

@@ -43,7 +43,16 @@ impl<'a> LexedStr<'a> {
                         res.was_joint();
                     }
                     res.push(kind);
+                    // Tag the token as joint if it is float with a fractional part
+                    // we use this jointness to inform the parser about what token split
+                    // event to emit when we encounter a float literal in a field access
+                    if kind == SyntaxKind::FLOAT_NUMBER {
+                        if !self.text(i).ends_with('.') {
+                            res.was_joint();
+                        }
+                    }
                 }
+
                 was_joint = true;
             }
         }
@@ -63,6 +72,9 @@ impl<'a> LexedStr<'a> {
                 Step::Token { kind, n_input_tokens: n_raw_tokens } => {
                     builder.token(kind, n_raw_tokens)
                 }
+                Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {
+                    builder.float_split(has_pseudo_dot)
+                }
                 Step::Enter { kind } => builder.enter(kind),
                 Step::Exit => builder.exit(),
                 Step::Error { msg } => {
@@ -109,6 +121,16 @@ impl Builder<'_, '_> {
         self.do_token(kind, n_tokens as usize);
     }
 
+    fn float_split(&mut self, has_pseudo_dot: bool) {
+        match mem::replace(&mut self.state, State::Normal) {
+            State::PendingEnter => unreachable!(),
+            State::PendingExit => (self.sink)(StrStep::Exit),
+            State::Normal => (),
+        }
+        self.eat_trivias();
+        self.do_float_split(has_pseudo_dot);
+    }
+
     fn enter(&mut self, kind: SyntaxKind) {
         match mem::replace(&mut self.state, State::Normal) {
             State::PendingEnter => {
@@ -164,6 +186,37 @@ impl Builder<'_, '_> {
         self.pos += n_tokens;
         (self.sink)(StrStep::Token { kind, text });
     }
+
+    fn do_float_split(&mut self, has_pseudo_dot: bool) {
+        let text = &self.lexed.range_text(self.pos..self.pos + 1);
+        self.pos += 1;
+        match text.split_once('.') {
+            Some((left, right)) => {
+                assert!(!left.is_empty());
+                (self.sink)(StrStep::Enter { kind: SyntaxKind::NAME_REF });
+                (self.sink)(StrStep::Token { kind: SyntaxKind::INT_NUMBER, text: left });
+                (self.sink)(StrStep::Exit);
+
+                // here we move the exit up, the original exit has been deleted in process
+                (self.sink)(StrStep::Exit);
+
+                (self.sink)(StrStep::Token { kind: SyntaxKind::DOT, text: "." });
+
+                if has_pseudo_dot {
+                    assert!(right.is_empty(), "{left}.{right}");
+                    self.state = State::Normal;
+                } else {
+                    (self.sink)(StrStep::Enter { kind: SyntaxKind::NAME_REF });
+                    (self.sink)(StrStep::Token { kind: SyntaxKind::INT_NUMBER, text: right });
+                    (self.sink)(StrStep::Exit);
+
+                    // the parser creates an unbalanced start node, we are required to close it here
+                    self.state = State::PendingExit;
+                }
+            }
+            None => unreachable!(),
+        }
+    }
 }
 
 fn n_attached_trivias<'a>(

crates/parser/src/tests/prefix_entries.rs

@@ -51,6 +51,9 @@ fn expr() {
     check(PrefixEntryPoint::Expr, "-1", "-1");
     check(PrefixEntryPoint::Expr, "fn foo() {}", "fn");
     check(PrefixEntryPoint::Expr, "#[attr] ()", "#[attr] ()");
+    check(PrefixEntryPoint::Expr, "foo.0", "foo.0");
+    check(PrefixEntryPoint::Expr, "foo.0.1", "foo.0.1");
+    check(PrefixEntryPoint::Expr, "foo.0. foo", "foo.0. foo");
 }
 
 #[test]
@@ -88,6 +91,7 @@ fn check(entry: PrefixEntryPoint, input: &str, prefix: &str) {
     for step in entry.parse(&input).iter() {
         match step {
             Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
+            Step::FloatSplit { .. } => n_tokens += 1,
             Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
         }
     }