Lexer start of line is false only for Mode::Expression (#8880)

## Summary

This PR fixes the bug in the lexer where the `Mode::Ipython` wasn't
being considered when initializing the soft keyword transformer which
wraps the lexer. This means that if the source code starts with either
`match` or `type` keyword, then the keywords were being considered as
name tokens instead. For example,

```python
match foo:
    case bar:
        pass
```

This would transform the `match` keyword into an identifier if the mode
is `Ipython`.

The fix is to reverse the condition in the soft keyword initializer so
that any new modes are by default considered as the lexer being at start
of line.

## Test Plan

Add a new test case for `Mode::Ipython` and verify the snapshot.

fixes: #8870
This commit is contained in:
Dhruv Manilawala 2023-11-28 14:38:25 -06:00 committed by GitHub
parent 9dee1883ce
commit 47d80f29a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 75 additions and 1 deletions

View file

@ -2160,6 +2160,14 @@ f"{(lambda x:{x})}"
assert_debug_snapshot!(lex_source(source));
}
#[test]
fn test_match_softkeyword_in_notebook() {
let source = r"match foo:
case bar:
pass";
assert_debug_snapshot!(lex_jupyter_source(source));
}
fn lex_fstring_error(source: &str) -> FStringErrorType {
match lex(source, Mode::Module).find_map(std::result::Result::err) {
Some(err) => match err.error {

View file

@ -0,0 +1,66 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(source)
---
[
(
Match,
0..5,
),
(
Name {
name: "foo",
},
6..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..15,
),
(
Case,
15..19,
),
(
Name {
name: "bar",
},
20..23,
),
(
Colon,
23..24,
),
(
Newline,
24..25,
),
(
Indent,
25..33,
),
(
Pass,
33..37,
),
(
Newline,
37..37,
),
(
Dedent,
37..37,
),
(
Dedent,
37..37,
),
]

View file

@ -31,7 +31,7 @@ where
pub fn new(lexer: I, mode: Mode) -> Self {
Self {
underlying: lexer.multipeek(), // spell-checker:ignore multipeek
start_of_line: matches!(mode, Mode::Module),
start_of_line: !matches!(mode, Mode::Expression),
}
}
}