mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-30 15:18:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use match statement in soft keyword token handler

Browse source
This commit is contained in:
Zanie 2023-07-14 12:11:59 -05:00
parent bbedb61607
commit d1c319ba15
1 changed files with 62 additions and 57 deletions
Download patch file Download diff file Expand all files Collapse all files

119
parser/src/soft_keywords.rs
View file

@ -50,71 +50,76 @@ where
// used as an identifier. We assume every soft keyword use is an identifier unless // used as an identifier. We assume every soft keyword use is an identifier unless
// a heuristic is met. // a heuristic is met.
// For `match` and `case`, all of the following conditions must be met: match tok {
// 1. The token is at the start of a logical line. // For `match` and `case`, all of the following conditions must be met:
// 2. The logical line contains a top-level colon (that is, a colon that is not nested // 1. The token is at the start of a logical line.
// inside a parenthesized expression, list, or dictionary). // 2. The logical line contains a top-level colon (that is, a colon that is not nested
// 3. The top-level colon is not the immediate sibling of a `match` or `case` token. // inside a parenthesized expression, list, or dictionary).
// (This is to avoid treating `match` or `case` as identifiers when annotated with // 3. The top-level colon is not the immediate sibling of a `match` or `case` token.
// type hints.) // (This is to avoid treating `match` or `case` as identifiers when annotated with
if matches!(tok, Tok::Match | Tok::Case) { // type hints.) type hints.)
if !self.start_of_line { Tok::Match | Tok::Case => {
next = Some(Ok((soft_to_name(tok), *range))); if !self.start_of_line {
} else { next = Some(Ok((soft_to_name(tok), *range)));
let mut nesting = 0; } else {
let mut first = true; let mut nesting = 0;
let mut seen_colon = false; let mut first = true;
let mut seen_lambda = false; let mut seen_colon = false;
while let Some(Ok((tok, _))) = self.underlying.peek() { let mut seen_lambda = false;
match tok { while let Some(Ok((tok, _))) = self.underlying.peek() {
Tok::Newline => break, match tok {
Tok::Lambda if nesting == 0 => seen_lambda = true, Tok::Newline => break,
Tok::Colon if nesting == 0 => { Tok::Lambda if nesting == 0 => seen_lambda = true,
if seen_lambda { Tok::Colon if nesting == 0 => {
seen_lambda = false; if seen_lambda {
} else if !first { seen_lambda = false;
seen_colon = true; } else if !first {
seen_colon = true;
}
} }
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
} }
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1, first = false;
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
} }
first = false; if !seen_colon {
} next = Some(Ok((soft_to_name(tok), *range)));
if !seen_colon {
next = Some(Ok((soft_to_name(tok), *range)));
}
}
}
// For `type` all of the following conditions must be met:
// 1. The token is at the start of a logical line.
// 2. The type token is followed by a name token.
// 3. The name token is followed by an equality token.
else if matches!(tok, Tok::Type) {
if !self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut nesting = 0;
let mut seen_name = false;
let mut seen_equal = false;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Name { .. } if nesting == 0 => seen_name = true,
// We treat a soft keyword token following a type token as a
// name to support cases like `type type = int` or `type match = int`
Tok::Type | Tok::Match | Tok::Case if nesting == 0 => seen_name = true,
Tok::Equal if nesting == 0 && seen_name => seen_equal = true,
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
} }
} }
if !(seen_name && seen_equal) { }
// For `type` all of the following conditions must be met:
// 1. The token is at the start of a logical line.
// 2. The type token is followed by a name token.
// 3. The name token is followed by an equality token.
Tok::Type => {
if !self.start_of_line {
next = Some(Ok((soft_to_name(tok), *range))); next = Some(Ok((soft_to_name(tok), *range)));
} else {
let mut nesting = 0;
let mut seen_name = false;
let mut seen_equal = false;
while let Some(Ok((tok, _))) = self.underlying.peek() {
match tok {
Tok::Newline => break,
Tok::Name { .. } if nesting == 0 => seen_name = true,
// We treat a soft keyword token following a type token as a
// name to support cases like `type type = int` or `type match = int`
Tok::Type | Tok::Match | Tok::Case if nesting == 0 => {
seen_name = true
}
Tok::Equal if nesting == 0 && seen_name => seen_equal = true,
Tok::Lpar | Tok::Lsqb | Tok::Lbrace => nesting += 1,
Tok::Rpar | Tok::Rsqb | Tok::Rbrace => nesting -= 1,
_ => {}
}
}
if !(seen_name && seen_equal) {
next = Some(Ok((soft_to_name(tok), *range)));
}
} }
} }
_ => (), // Not a soft keyword token
} }
} }