Split float literal tokens at the .

This commit is contained in:
Jonas Schievink 2022-05-04 16:51:05 +02:00
parent 502c519e7d
commit 1bc3305d95
23 changed files with 157 additions and 72 deletions

View file

@ -318,7 +318,7 @@ fn name_ref(p: &mut Parser) {
}
fn name_ref_or_index(p: &mut Parser) {
assert!(p.at(IDENT) || p.at(INT_NUMBER));
assert!(p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART));
let m = p.start();
p.bump_any();
m.complete(p, NAME_REF);

View file

@ -475,11 +475,8 @@ fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker {
assert!(p.at(T![.]));
let m = lhs.precede(p);
p.bump(T![.]);
if p.at(IDENT) || p.at(INT_NUMBER) {
if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) {
name_ref_or_index(p);
} else if p.at(FLOAT_NUMBER) {
// FIXME: How to recover and instead parse INT + T![.]?
p.bump_any();
} else {
p.error("expected field name or number");
}

View file

@ -17,7 +17,7 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
T![true],
T![false],
INT_NUMBER,
FLOAT_NUMBER,
FLOAT_NUMBER_PART,
BYTE,
CHAR,
STRING,
@ -29,11 +29,19 @@ pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> {
return None;
}
let m = p.start();
if p.at(FLOAT_NUMBER) {
if p.at(FLOAT_NUMBER_PART) {
// Floats can be up to 3 tokens: 2 `FLOAT_NUMBER_PART`s separated by 1 `DOT`
let f = p.start();
p.bump(FLOAT_NUMBER);
p.bump(FLOAT_NUMBER_PART);
if p.at(DOT) {
p.bump(DOT);
if p.at(FLOAT_NUMBER_PART) {
p.bump(FLOAT_NUMBER_PART);
}
}
f.complete(p, FLOAT_LITERAL);
} else {
// Everything else is just one token.
p.bump_any();
}
Some(m.complete(p, LITERAL))

View file

@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> {
}
fn is_literal_pat_start(p: &Parser) -> bool {
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER)
p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART)
|| p.at_ts(expressions::LITERAL_FIRST)
}

View file

@ -177,7 +177,7 @@ impl<'a> Converter<'a> {
rustc_lexer::TokenKind::RawIdent => IDENT,
rustc_lexer::TokenKind::Literal { kind, .. } => {
self.extend_literal(token_text.len(), kind);
self.extend_literal(token_text, kind);
return;
}
@ -223,7 +223,7 @@ impl<'a> Converter<'a> {
self.push(syntax_kind, token_text.len(), err);
}
fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) {
fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) {
let mut err = "";
let syntax_kind = match *kind {
@ -237,7 +237,22 @@ impl<'a> Converter<'a> {
if empty_exponent {
err = "Missing digits after the exponent symbol";
}
FLOAT_NUMBER
// In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0`
// is lexed as a float, we split floats that contain a `.` into 3 tokens.
if let Some((before, after)) = token_text.split_once('.') {
let err = if err.is_empty() { None } else { Some(err) };
if !before.is_empty() {
self.push(FLOAT_NUMBER_PART, before.len(), None);
}
self.push(DOT, 1, None);
if !after.is_empty() {
self.push(FLOAT_NUMBER_PART, after.len(), err);
}
return;
}
FLOAT_NUMBER_PART
}
rustc_lexer::LiteralKind::Char { terminated } => {
if !terminated {
@ -295,6 +310,6 @@ impl<'a> Converter<'a> {
};
let err = if err.is_empty() { None } else { Some(err) };
self.push(syntax_kind, len, err);
self.push(syntax_kind, token_text.len(), err);
}
}

File diff suppressed because one or more lines are too long