Add range to lexer test snapshots (#7265)

## Summary

This PR updates the lexer test snapshots to include the range value as
well. This is mainly a mechanical refactor.

### Motivation

The main motivation is so that we can verify that the ranges are valid
and do not overlap.

## Test Plan

`cargo test`
This commit is contained in:
Dhruv Manilawala 2023-09-12 00:42:46 +05:30 committed by GitHub
parent 24b848a4ea
commit a41bb2733f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
44 changed files with 2197 additions and 871 deletions

View file

@ -1273,17 +1273,20 @@ mod tests {
const MAC_EOL: &str = "\r";
const UNIX_EOL: &str = "\n";
fn lex_source(source: &str) -> Vec<Tok> {
let lexer = lex(source, Mode::Module);
lexer.map(|result| result.unwrap().0).collect()
fn lex_source_with_mode(source: &str, mode: Mode) -> Vec<Spanned> {
let lexer = lex(source, mode);
lexer.map(std::result::Result::unwrap).collect()
}
fn lex_jupyter_source(source: &str) -> Vec<Tok> {
let lexer = lex(source, Mode::Ipython);
lexer.map(|x| x.unwrap().0).collect()
fn lex_source(source: &str) -> Vec<Spanned> {
lex_source_with_mode(source, Mode::Module)
}
fn ipython_escape_command_line_continuation_eol(eol: &str) -> Vec<Tok> {
fn lex_jupyter_source(source: &str) -> Vec<Spanned> {
lex_source_with_mode(source, Mode::Ipython)
}
fn ipython_escape_command_line_continuation_eol(eol: &str) -> Vec<Spanned> {
let source = format!("%matplotlib \\{eol} --inline");
lex_jupyter_source(&source)
}
@ -1303,7 +1306,7 @@ mod tests {
assert_debug_snapshot!(ipython_escape_command_line_continuation_eol(WINDOWS_EOL));
}
fn ipython_escape_command_line_continuation_with_eol_and_eof(eol: &str) -> Vec<Tok> {
fn ipython_escape_command_line_continuation_with_eol_and_eof(eol: &str) -> Vec<Spanned> {
let source = format!("%matplotlib \\{eol}");
lex_jupyter_source(&source)
}
@ -1403,8 +1406,8 @@ baz = %matplotlib \
assert_debug_snapshot!(lex_jupyter_source(source));
}
fn assert_no_ipython_escape_command(tokens: &[Tok]) {
for tok in tokens {
fn assert_no_ipython_escape_command(tokens: &[Spanned]) {
for (tok, _) in tokens {
if let Tok::IpyEscapeCommand { .. } = tok {
panic!("Unexpected escape command token: {tok:?}")
}
@ -1458,7 +1461,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(lex_source(&source));
}
fn comment_until_eol(eol: &str) -> Vec<Tok> {
fn comment_until_eol(eol: &str) -> Vec<Spanned> {
let source = format!("123 # Foo{eol}456");
lex_source(&source)
}
@ -1484,7 +1487,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(lex_source(source));
}
fn indentation_with_eol(eol: &str) -> Vec<Tok> {
fn indentation_with_eol(eol: &str) -> Vec<Spanned> {
let source = format!("def foo():{eol} return 99{eol}{eol}");
lex_source(&source)
}
@ -1504,7 +1507,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(indentation_with_eol(WINDOWS_EOL));
}
fn double_dedent_with_eol(eol: &str) -> Vec<Tok> {
fn double_dedent_with_eol(eol: &str) -> Vec<Spanned> {
let source = format!("def foo():{eol} if x:{eol}{eol} return 99{eol}{eol}");
lex_source(&source)
}
@ -1524,7 +1527,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(double_dedent_with_eol(WINDOWS_EOL));
}
fn double_dedent_with_tabs_eol(eol: &str) -> Vec<Tok> {
fn double_dedent_with_tabs_eol(eol: &str) -> Vec<Spanned> {
let source = format!("def foo():{eol}\tif x:{eol}{eol}\t\t return 99{eol}{eol}");
lex_source(&source)
}
@ -1544,7 +1547,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(double_dedent_with_tabs_eol(WINDOWS_EOL));
}
fn newline_in_brackets_eol(eol: &str) -> Vec<Tok> {
fn newline_in_brackets_eol(eol: &str) -> Vec<Spanned> {
let source = r"x = [
1,2
@ -1604,7 +1607,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(lex_source(source));
}
fn string_continuation_with_eol(eol: &str) -> Vec<Tok> {
fn string_continuation_with_eol(eol: &str) -> Vec<Spanned> {
let source = format!("\"abc\\{eol}def\"");
lex_source(&source)
}
@ -1630,7 +1633,7 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(lex_source(source));
}
fn triple_quoted_eol(eol: &str) -> Vec<Tok> {
fn triple_quoted_eol(eol: &str) -> Vec<Spanned> {
let source = format!("\"\"\"{eol} test string{eol} \"\"\"");
lex_source(&source)
}

View file

@ -3,20 +3,44 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
Name {
name: "a_variable",
},
Equal,
Int {
value: 99,
},
Plus,
Int {
value: 2,
},
Minus,
Int {
value: 0,
},
Newline,
(
Name {
name: "a_variable",
},
0..10,
),
(
Equal,
11..12,
),
(
Int {
value: 99,
},
13..15,
),
(
Plus,
16..17,
),
(
Int {
value: 2,
},
18..19,
),
(
Minus,
19..20,
),
(
Int {
value: 0,
},
20..21,
),
(
Newline,
21..21,
),
]

View file

@ -1,17 +1,32 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: comment_until_eol(MAC_EOL)
---
[
Int {
value: 123,
},
Comment(
"# Foo",
(
Int {
value: 123,
},
0..3,
),
(
Comment(
"# Foo",
),
5..10,
),
(
Newline,
10..11,
),
(
Int {
value: 456,
},
11..14,
),
(
Newline,
14..14,
),
Newline,
Int {
value: 456,
},
Newline,
]

View file

@ -1,17 +1,32 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: comment_until_eol(UNIX_EOL)
---
[
Int {
value: 123,
},
Comment(
"# Foo",
(
Int {
value: 123,
},
0..3,
),
(
Comment(
"# Foo",
),
5..10,
),
(
Newline,
10..11,
),
(
Int {
value: 456,
},
11..14,
),
(
Newline,
14..14,
),
Newline,
Int {
value: 456,
},
Newline,
]

View file

@ -1,17 +1,32 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: comment_until_eol(WINDOWS_EOL)
---
[
Int {
value: 123,
},
Comment(
"# Foo",
(
Int {
value: 123,
},
0..3,
),
(
Comment(
"# Foo",
),
5..10,
),
(
Newline,
10..12,
),
(
Int {
value: 456,
},
12..15,
),
(
Newline,
15..15,
),
Newline,
Int {
value: 456,
},
Newline,
]

View file

@ -1,31 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: double_dedent_with_eol(MAC_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
If,
Name {
name: "x",
},
Colon,
Newline,
NonLogicalNewline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..12,
),
(
If,
12..14,
),
(
Name {
name: "x",
},
15..16,
),
(
Colon,
16..17,
),
(
Newline,
17..18,
),
(
NonLogicalNewline,
18..19,
),
(
Indent,
19..21,
),
(
Return,
21..27,
),
(
Int {
value: 99,
},
28..30,
),
(
Newline,
30..31,
),
(
NonLogicalNewline,
31..32,
),
(
Dedent,
32..32,
),
(
Dedent,
32..32,
),
]

View file

@ -1,31 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: double_dedent_with_tabs_eol(MAC_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
If,
Name {
name: "x",
},
Colon,
Newline,
NonLogicalNewline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..12,
),
(
If,
12..14,
),
(
Name {
name: "x",
},
15..16,
),
(
Colon,
16..17,
),
(
Newline,
17..18,
),
(
NonLogicalNewline,
18..19,
),
(
Indent,
19..22,
),
(
Return,
22..28,
),
(
Int {
value: 99,
},
29..31,
),
(
Newline,
31..32,
),
(
NonLogicalNewline,
32..33,
),
(
Dedent,
33..33,
),
(
Dedent,
33..33,
),
]

View file

@ -1,31 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: double_dedent_with_tabs_eol(UNIX_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
If,
Name {
name: "x",
},
Colon,
Newline,
NonLogicalNewline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..12,
),
(
If,
12..14,
),
(
Name {
name: "x",
},
15..16,
),
(
Colon,
16..17,
),
(
Newline,
17..18,
),
(
NonLogicalNewline,
18..19,
),
(
Indent,
19..22,
),
(
Return,
22..28,
),
(
Int {
value: 99,
},
29..31,
),
(
Newline,
31..32,
),
(
NonLogicalNewline,
32..33,
),
(
Dedent,
33..33,
),
(
Dedent,
33..33,
),
]

View file

@ -1,31 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: double_dedent_with_tabs_eol(WINDOWS_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
If,
Name {
name: "x",
},
Colon,
Newline,
NonLogicalNewline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..12,
),
(
Indent,
12..13,
),
(
If,
13..15,
),
(
Name {
name: "x",
},
16..17,
),
(
Colon,
17..18,
),
(
Newline,
18..20,
),
(
NonLogicalNewline,
20..22,
),
(
Indent,
22..25,
),
(
Return,
25..31,
),
(
Int {
value: 99,
},
32..34,
),
(
Newline,
34..36,
),
(
NonLogicalNewline,
36..38,
),
(
Dedent,
38..38,
),
(
Dedent,
38..38,
),
]

View file

@ -1,31 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: double_dedent_with_eol(UNIX_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
If,
Name {
name: "x",
},
Colon,
Newline,
NonLogicalNewline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..12,
),
(
If,
12..14,
),
(
Name {
name: "x",
},
15..16,
),
(
Colon,
16..17,
),
(
Newline,
17..18,
),
(
NonLogicalNewline,
18..19,
),
(
Indent,
19..21,
),
(
Return,
21..27,
),
(
Int {
value: 99,
},
28..30,
),
(
Newline,
30..31,
),
(
NonLogicalNewline,
31..32,
),
(
Dedent,
32..32,
),
(
Dedent,
32..32,
),
]

View file

@ -1,31 +1,88 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: double_dedent_with_eol(WINDOWS_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
If,
Name {
name: "x",
},
Colon,
Newline,
NonLogicalNewline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..12,
),
(
Indent,
12..13,
),
(
If,
13..15,
),
(
Name {
name: "x",
},
16..17,
),
(
Colon,
17..18,
),
(
Newline,
18..20,
),
(
NonLogicalNewline,
20..22,
),
(
Indent,
22..24,
),
(
Return,
24..30,
),
(
Int {
value: 99,
},
31..33,
),
(
Newline,
33..35,
),
(
NonLogicalNewline,
35..37,
),
(
Dedent,
37..37,
),
(
Dedent,
37..37,
),
]

View file

@ -3,49 +3,103 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(source)
---
[
IpyEscapeCommand {
value: "",
kind: Magic,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Magic2,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Shell,
},
Newline,
IpyEscapeCommand {
value: "",
kind: ShCap,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Paren,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Quote,
},
Newline,
IpyEscapeCommand {
value: "",
kind: Quote2,
},
Newline,
(
IpyEscapeCommand {
value: "",
kind: Magic,
},
0..1,
),
(
Newline,
1..2,
),
(
IpyEscapeCommand {
value: "",
kind: Magic2,
},
2..4,
),
(
Newline,
4..5,
),
(
IpyEscapeCommand {
value: "",
kind: Shell,
},
5..6,
),
(
Newline,
6..7,
),
(
IpyEscapeCommand {
value: "",
kind: ShCap,
},
7..9,
),
(
Newline,
9..10,
),
(
IpyEscapeCommand {
value: "",
kind: Help,
},
10..11,
),
(
Newline,
11..12,
),
(
IpyEscapeCommand {
value: "",
kind: Help2,
},
12..14,
),
(
Newline,
14..15,
),
(
IpyEscapeCommand {
value: "",
kind: Paren,
},
15..16,
),
(
Newline,
16..17,
),
(
IpyEscapeCommand {
value: "",
kind: Quote,
},
17..18,
),
(
Newline,
18..19,
),
(
IpyEscapeCommand {
value: "",
kind: Quote2,
},
19..20,
),
(
Newline,
20..20,
),
]

View file

@ -3,10 +3,16 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
String {
value: "\\N{EN SPACE}",
kind: String,
triple_quoted: false,
},
Newline,
(
String {
value: "\\N{EN SPACE}",
kind: String,
triple_quoted: false,
},
0..14,
),
(
Newline,
14..14,
),
]

View file

@ -1,22 +1,58 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: indentation_with_eol(MAC_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..15,
),
(
Return,
15..21,
),
(
Int {
value: 99,
},
22..24,
),
(
Newline,
24..25,
),
(
NonLogicalNewline,
25..26,
),
(
Dedent,
26..26,
),
]

View file

@ -1,22 +1,58 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: indentation_with_eol(UNIX_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..11,
),
(
Indent,
11..15,
),
(
Return,
15..21,
),
(
Int {
value: 99,
},
22..24,
),
(
Newline,
24..25,
),
(
NonLogicalNewline,
25..26,
),
(
Dedent,
26..26,
),
]

View file

@ -1,22 +1,58 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: indentation_with_eol(WINDOWS_EOL)
---
[
Def,
Name {
name: "foo",
},
Lpar,
Rpar,
Colon,
Newline,
Indent,
Return,
Int {
value: 99,
},
Newline,
NonLogicalNewline,
Dedent,
(
Def,
0..3,
),
(
Name {
name: "foo",
},
4..7,
),
(
Lpar,
7..8,
),
(
Rpar,
8..9,
),
(
Colon,
9..10,
),
(
Newline,
10..12,
),
(
Indent,
12..16,
),
(
Return,
16..22,
),
(
Int {
value: 99,
},
23..25,
),
(
Newline,
25..27,
),
(
NonLogicalNewline,
27..29,
),
(
Dedent,
29..29,
),
]

View file

@ -3,59 +3,125 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(source)
---
[
IpyEscapeCommand {
value: "foo",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "timeit a = b",
kind: Magic,
},
Newline,
IpyEscapeCommand {
value: "timeit a % 3",
kind: Magic,
},
Newline,
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
Newline,
IpyEscapeCommand {
value: "pwd && ls -a | sed 's/^/\\\\ /'",
kind: Shell,
},
Newline,
IpyEscapeCommand {
value: "cd /Users/foo/Library/Application\\ Support/",
kind: ShCap,
},
Newline,
IpyEscapeCommand {
value: "foo 1 2",
kind: Paren,
},
Newline,
IpyEscapeCommand {
value: "foo 1 2",
kind: Quote,
},
Newline,
IpyEscapeCommand {
value: "foo 1 2",
kind: Quote2,
},
Newline,
IpyEscapeCommand {
value: "ls",
kind: Shell,
},
Newline,
(
IpyEscapeCommand {
value: "foo",
kind: Help,
},
0..4,
),
(
Newline,
4..5,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
5..10,
),
(
Newline,
10..11,
),
(
IpyEscapeCommand {
value: "timeit a = b",
kind: Magic,
},
11..24,
),
(
Newline,
24..25,
),
(
IpyEscapeCommand {
value: "timeit a % 3",
kind: Magic,
},
25..38,
),
(
Newline,
38..39,
),
(
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
39..65,
),
(
Newline,
65..66,
),
(
IpyEscapeCommand {
value: "pwd && ls -a | sed 's/^/\\\\ /'",
kind: Shell,
},
66..103,
),
(
Newline,
103..104,
),
(
IpyEscapeCommand {
value: "cd /Users/foo/Library/Application\\ Support/",
kind: ShCap,
},
104..149,
),
(
Newline,
149..150,
),
(
IpyEscapeCommand {
value: "foo 1 2",
kind: Paren,
},
150..158,
),
(
Newline,
158..159,
),
(
IpyEscapeCommand {
value: "foo 1 2",
kind: Quote,
},
159..167,
),
(
Newline,
167..168,
),
(
IpyEscapeCommand {
value: "foo 1 2",
kind: Quote2,
},
168..176,
),
(
Newline,
176..177,
),
(
IpyEscapeCommand {
value: "ls",
kind: Shell,
},
177..180,
),
(
Newline,
180..180,
),
]

View file

@ -3,40 +3,88 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(source)
---
[
Name {
name: "pwd",
},
Equal,
IpyEscapeCommand {
value: "pwd",
kind: Shell,
},
Newline,
Name {
name: "foo",
},
Equal,
IpyEscapeCommand {
value: "timeit a = b",
kind: Magic,
},
Newline,
Name {
name: "bar",
},
Equal,
IpyEscapeCommand {
value: "timeit a % 3",
kind: Magic,
},
Newline,
Name {
name: "baz",
},
Equal,
IpyEscapeCommand {
value: "matplotlib inline",
kind: Magic,
},
Newline,
(
Name {
name: "pwd",
},
0..3,
),
(
Equal,
4..5,
),
(
IpyEscapeCommand {
value: "pwd",
kind: Shell,
},
6..10,
),
(
Newline,
10..11,
),
(
Name {
name: "foo",
},
11..14,
),
(
Equal,
15..16,
),
(
IpyEscapeCommand {
value: "timeit a = b",
kind: Magic,
},
17..30,
),
(
Newline,
30..31,
),
(
Name {
name: "bar",
},
31..34,
),
(
Equal,
35..36,
),
(
IpyEscapeCommand {
value: "timeit a % 3",
kind: Magic,
},
37..50,
),
(
Newline,
50..51,
),
(
Name {
name: "baz",
},
51..54,
),
(
Equal,
55..56,
),
(
IpyEscapeCommand {
value: "matplotlib inline",
kind: Magic,
},
57..85,
),
(
Newline,
85..85,
),
]

View file

@ -3,15 +3,39 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(source)
---
[
If,
True,
Colon,
Newline,
Indent,
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
Newline,
Dedent,
(
If,
0..2,
),
(
True,
3..7,
),
(
Colon,
7..8,
),
(
Newline,
8..9,
),
(
Indent,
9..13,
),
(
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
13..43,
),
(
Newline,
43..43,
),
(
Dedent,
43..43,
),
]

View file

@ -1,11 +1,17 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(&source)
expression: ipython_escape_command_line_continuation_eol(MAC_EOL)
---
[
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
Newline,
(
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
0..24,
),
(
Newline,
24..24,
),
]

View file

@ -1,11 +1,17 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(&source)
expression: ipython_escape_command_line_continuation_eol(UNIX_EOL)
---
[
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
Newline,
(
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
0..24,
),
(
Newline,
24..24,
),
]

View file

@ -1,11 +1,17 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(&source)
expression: ipython_escape_command_line_continuation_eol(WINDOWS_EOL)
---
[
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
Newline,
(
IpyEscapeCommand {
value: "matplotlib --inline",
kind: Magic,
},
0..25,
),
(
Newline,
25..25,
),
]

View file

@ -1,11 +1,17 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(&source)
expression: ipython_escape_command_line_continuation_with_eol_and_eof(MAC_EOL)
---
[
IpyEscapeCommand {
value: "matplotlib ",
kind: Magic,
},
Newline,
(
IpyEscapeCommand {
value: "matplotlib ",
kind: Magic,
},
0..14,
),
(
Newline,
14..14,
),
]

View file

@ -1,11 +1,17 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(&source)
expression: ipython_escape_command_line_continuation_with_eol_and_eof(UNIX_EOL)
---
[
IpyEscapeCommand {
value: "matplotlib ",
kind: Magic,
},
Newline,
(
IpyEscapeCommand {
value: "matplotlib ",
kind: Magic,
},
0..14,
),
(
Newline,
14..14,
),
]

View file

@ -1,11 +1,17 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(&source)
expression: ipython_escape_command_line_continuation_with_eol_and_eof(WINDOWS_EOL)
---
[
IpyEscapeCommand {
value: "matplotlib ",
kind: Magic,
},
Newline,
(
IpyEscapeCommand {
value: "matplotlib ",
kind: Magic,
},
0..15,
),
(
Newline,
15..15,
),
]

View file

@ -3,84 +3,180 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_jupyter_source(source)
---
[
IpyEscapeCommand {
value: "foo",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: " foo ?",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "foo???",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "?foo???",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "foo",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: " ?",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "??",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "%foo",
kind: Help,
},
Newline,
IpyEscapeCommand {
value: "%foo",
kind: Help2,
},
Newline,
IpyEscapeCommand {
value: "foo???",
kind: Magic2,
},
Newline,
IpyEscapeCommand {
value: "pwd",
kind: Help,
},
Newline,
(
IpyEscapeCommand {
value: "foo",
kind: Help,
},
0..5,
),
(
Newline,
5..6,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help,
},
6..15,
),
(
Newline,
15..16,
),
(
IpyEscapeCommand {
value: " foo ?",
kind: Help2,
},
16..27,
),
(
Newline,
27..28,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
28..34,
),
(
Newline,
34..35,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
35..42,
),
(
Newline,
42..43,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help,
},
43..50,
),
(
Newline,
50..51,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help2,
},
51..59,
),
(
Newline,
59..60,
),
(
IpyEscapeCommand {
value: "foo???",
kind: Help2,
},
60..68,
),
(
Newline,
68..69,
),
(
IpyEscapeCommand {
value: "?foo???",
kind: Help2,
},
69..78,
),
(
Newline,
78..79,
),
(
IpyEscapeCommand {
value: "foo",
kind: Help,
},
79..92,
),
(
Newline,
92..93,
),
(
IpyEscapeCommand {
value: " ?",
kind: Help2,
},
93..99,
),
(
Newline,
99..100,
),
(
IpyEscapeCommand {
value: "??",
kind: Help2,
},
100..104,
),
(
Newline,
104..105,
),
(
IpyEscapeCommand {
value: "%foo",
kind: Help,
},
105..110,
),
(
Newline,
110..111,
),
(
IpyEscapeCommand {
value: "%foo",
kind: Help2,
},
111..117,
),
(
Newline,
117..118,
),
(
IpyEscapeCommand {
value: "foo???",
kind: Magic2,
},
118..126,
),
(
Newline,
126..127,
),
(
IpyEscapeCommand {
value: "pwd",
kind: Help,
},
127..132,
),
(
Newline,
132..132,
),
]

View file

@ -3,11 +3,20 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
---
[
Int {
value: 99232,
},
Comment(
"#",
(
Int {
value: 99232,
},
0..5,
),
(
Comment(
"#",
),
7..8,
),
(
Newline,
8..8,
),
Newline,
]

View file

@ -3,11 +3,20 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
---
[
Int {
value: 99232,
},
Comment(
"# foo",
(
Int {
value: 99232,
},
0..5,
),
(
Comment(
"# foo",
),
7..12,
),
(
Newline,
12..12,
),
Newline,
]

View file

@ -3,11 +3,20 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
---
[
Int {
value: 99232,
},
Comment(
"# ",
(
Int {
value: 99232,
},
0..5,
),
(
Comment(
"# ",
),
7..9,
),
(
Newline,
9..9,
),
Newline,
]

View file

@ -3,11 +3,20 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
---
[
Int {
value: 99232,
},
Comment(
"# ",
(
Int {
value: 99232,
},
0..5,
),
(
Comment(
"# ",
),
7..10,
),
(
Newline,
10..10,
),
Newline,
]

View file

@ -3,12 +3,24 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
Comment(
"#Hello",
(
Comment(
"#Hello",
),
0..6,
),
NonLogicalNewline,
Comment(
"#World",
(
NonLogicalNewline,
6..7,
),
(
Comment(
"#World",
),
7..13,
),
(
NonLogicalNewline,
13..14,
),
NonLogicalNewline,
]

View file

@ -1,52 +1,142 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: newline_in_brackets_eol(MAC_EOL)
---
[
Name {
name: "x",
},
Equal,
Lsqb,
NonLogicalNewline,
NonLogicalNewline,
Int {
value: 1,
},
Comma,
Int {
value: 2,
},
NonLogicalNewline,
Comma,
Lpar,
Int {
value: 3,
},
Comma,
NonLogicalNewline,
Int {
value: 4,
},
Comma,
NonLogicalNewline,
Rpar,
Comma,
Lbrace,
NonLogicalNewline,
Int {
value: 5,
},
Comma,
NonLogicalNewline,
Int {
value: 6,
},
Comma,
Int {
value: 7,
},
Rbrace,
Rsqb,
Newline,
(
Name {
name: "x",
},
0..1,
),
(
Equal,
2..3,
),
(
Lsqb,
4..5,
),
(
NonLogicalNewline,
5..6,
),
(
NonLogicalNewline,
6..7,
),
(
Int {
value: 1,
},
11..12,
),
(
Comma,
12..13,
),
(
Int {
value: 2,
},
13..14,
),
(
NonLogicalNewline,
14..15,
),
(
Comma,
15..16,
),
(
Lpar,
16..17,
),
(
Int {
value: 3,
},
17..18,
),
(
Comma,
18..19,
),
(
NonLogicalNewline,
19..20,
),
(
Int {
value: 4,
},
20..21,
),
(
Comma,
21..22,
),
(
NonLogicalNewline,
22..23,
),
(
Rpar,
23..24,
),
(
Comma,
24..25,
),
(
Lbrace,
26..27,
),
(
NonLogicalNewline,
27..28,
),
(
Int {
value: 5,
},
28..29,
),
(
Comma,
29..30,
),
(
NonLogicalNewline,
30..31,
),
(
Int {
value: 6,
},
31..32,
),
(
Comma,
32..33,
),
(
Int {
value: 7,
},
35..36,
),
(
Rbrace,
36..37,
),
(
Rsqb,
37..38,
),
(
Newline,
38..39,
),
]

View file

@ -1,52 +1,142 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: newline_in_brackets_eol(UNIX_EOL)
---
[
Name {
name: "x",
},
Equal,
Lsqb,
NonLogicalNewline,
NonLogicalNewline,
Int {
value: 1,
},
Comma,
Int {
value: 2,
},
NonLogicalNewline,
Comma,
Lpar,
Int {
value: 3,
},
Comma,
NonLogicalNewline,
Int {
value: 4,
},
Comma,
NonLogicalNewline,
Rpar,
Comma,
Lbrace,
NonLogicalNewline,
Int {
value: 5,
},
Comma,
NonLogicalNewline,
Int {
value: 6,
},
Comma,
Int {
value: 7,
},
Rbrace,
Rsqb,
Newline,
(
Name {
name: "x",
},
0..1,
),
(
Equal,
2..3,
),
(
Lsqb,
4..5,
),
(
NonLogicalNewline,
5..6,
),
(
NonLogicalNewline,
6..7,
),
(
Int {
value: 1,
},
11..12,
),
(
Comma,
12..13,
),
(
Int {
value: 2,
},
13..14,
),
(
NonLogicalNewline,
14..15,
),
(
Comma,
15..16,
),
(
Lpar,
16..17,
),
(
Int {
value: 3,
},
17..18,
),
(
Comma,
18..19,
),
(
NonLogicalNewline,
19..20,
),
(
Int {
value: 4,
},
20..21,
),
(
Comma,
21..22,
),
(
NonLogicalNewline,
22..23,
),
(
Rpar,
23..24,
),
(
Comma,
24..25,
),
(
Lbrace,
26..27,
),
(
NonLogicalNewline,
27..28,
),
(
Int {
value: 5,
},
28..29,
),
(
Comma,
29..30,
),
(
NonLogicalNewline,
30..31,
),
(
Int {
value: 6,
},
31..32,
),
(
Comma,
32..33,
),
(
Int {
value: 7,
},
35..36,
),
(
Rbrace,
36..37,
),
(
Rsqb,
37..38,
),
(
Newline,
38..39,
),
]

View file

@ -1,52 +1,142 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: newline_in_brackets_eol(WINDOWS_EOL)
---
[
Name {
name: "x",
},
Equal,
Lsqb,
NonLogicalNewline,
NonLogicalNewline,
Int {
value: 1,
},
Comma,
Int {
value: 2,
},
NonLogicalNewline,
Comma,
Lpar,
Int {
value: 3,
},
Comma,
NonLogicalNewline,
Int {
value: 4,
},
Comma,
NonLogicalNewline,
Rpar,
Comma,
Lbrace,
NonLogicalNewline,
Int {
value: 5,
},
Comma,
NonLogicalNewline,
Int {
value: 6,
},
Comma,
Int {
value: 7,
},
Rbrace,
Rsqb,
Newline,
(
Name {
name: "x",
},
0..1,
),
(
Equal,
2..3,
),
(
Lsqb,
4..5,
),
(
NonLogicalNewline,
5..7,
),
(
NonLogicalNewline,
7..9,
),
(
Int {
value: 1,
},
13..14,
),
(
Comma,
14..15,
),
(
Int {
value: 2,
},
15..16,
),
(
NonLogicalNewline,
16..18,
),
(
Comma,
18..19,
),
(
Lpar,
19..20,
),
(
Int {
value: 3,
},
20..21,
),
(
Comma,
21..22,
),
(
NonLogicalNewline,
22..24,
),
(
Int {
value: 4,
},
24..25,
),
(
Comma,
25..26,
),
(
NonLogicalNewline,
26..28,
),
(
Rpar,
28..29,
),
(
Comma,
29..30,
),
(
Lbrace,
31..32,
),
(
NonLogicalNewline,
32..34,
),
(
Int {
value: 5,
},
34..35,
),
(
Comma,
35..36,
),
(
NonLogicalNewline,
36..38,
),
(
Int {
value: 6,
},
38..39,
),
(
Comma,
39..40,
),
(
Int {
value: 7,
},
43..44,
),
(
Rbrace,
44..45,
),
(
Rsqb,
45..46,
),
(
Newline,
46..48,
),
]

View file

@ -3,32 +3,68 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
Lpar,
NonLogicalNewline,
String {
value: "a",
kind: String,
triple_quoted: false,
},
NonLogicalNewline,
String {
value: "b",
kind: String,
triple_quoted: false,
},
NonLogicalNewline,
NonLogicalNewline,
String {
value: "c",
kind: String,
triple_quoted: false,
},
String {
value: "d",
kind: String,
triple_quoted: false,
},
NonLogicalNewline,
Rpar,
Newline,
(
Lpar,
0..1,
),
(
NonLogicalNewline,
1..2,
),
(
String {
value: "a",
kind: String,
triple_quoted: false,
},
6..9,
),
(
NonLogicalNewline,
9..10,
),
(
String {
value: "b",
kind: String,
triple_quoted: false,
},
14..17,
),
(
NonLogicalNewline,
17..18,
),
(
NonLogicalNewline,
18..19,
),
(
String {
value: "c",
kind: String,
triple_quoted: false,
},
23..26,
),
(
String {
value: "d",
kind: String,
triple_quoted: false,
},
33..36,
),
(
NonLogicalNewline,
36..37,
),
(
Rpar,
37..38,
),
(
Newline,
38..38,
),
]

View file

@ -3,40 +3,76 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
Int {
value: 47,
},
Int {
value: 10,
},
Int {
value: 13,
},
Int {
value: 0,
},
Int {
value: 123,
},
Int {
value: 1234567890,
},
Float {
value: 0.2,
},
Float {
value: 100.0,
},
Float {
value: 2100.0,
},
Complex {
real: 0.0,
imag: 2.0,
},
Complex {
real: 0.0,
imag: 2.2,
},
Newline,
(
Int {
value: 47,
},
0..4,
),
(
Int {
value: 10,
},
5..9,
),
(
Int {
value: 13,
},
10..16,
),
(
Int {
value: 0,
},
17..18,
),
(
Int {
value: 123,
},
19..22,
),
(
Int {
value: 1234567890,
},
23..36,
),
(
Float {
value: 0.2,
},
37..40,
),
(
Float {
value: 100.0,
},
41..45,
),
(
Float {
value: 2100.0,
},
46..51,
),
(
Complex {
real: 0.0,
imag: 2.0,
},
52..54,
),
(
Complex {
real: 0.0,
imag: 2.2,
},
55..59,
),
(
Newline,
59..59,
),
]

View file

@ -3,10 +3,28 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
DoubleSlash,
DoubleSlash,
DoubleSlashEqual,
Slash,
Slash,
Newline,
(
DoubleSlash,
0..2,
),
(
DoubleSlash,
2..4,
),
(
DoubleSlashEqual,
4..7,
),
(
Slash,
7..8,
),
(
Slash,
9..10,
),
(
Newline,
10..10,
),
]

View file

@ -3,50 +3,80 @@ source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(source)
---
[
String {
value: "double",
kind: String,
triple_quoted: false,
},
String {
value: "single",
kind: String,
triple_quoted: false,
},
String {
value: "can\\'t",
kind: String,
triple_quoted: false,
},
String {
value: "\\\\\\\"",
kind: String,
triple_quoted: false,
},
String {
value: "\\t\\r\\n",
kind: String,
triple_quoted: false,
},
String {
value: "\\g",
kind: String,
triple_quoted: false,
},
String {
value: "raw\\'",
kind: RawString,
triple_quoted: false,
},
String {
value: "\\420",
kind: String,
triple_quoted: false,
},
String {
value: "\\200\\0a",
kind: String,
triple_quoted: false,
},
Newline,
(
String {
value: "double",
kind: String,
triple_quoted: false,
},
0..8,
),
(
String {
value: "single",
kind: String,
triple_quoted: false,
},
9..17,
),
(
String {
value: "can\\'t",
kind: String,
triple_quoted: false,
},
18..26,
),
(
String {
value: "\\\\\\\"",
kind: String,
triple_quoted: false,
},
27..33,
),
(
String {
value: "\\t\\r\\n",
kind: String,
triple_quoted: false,
},
34..42,
),
(
String {
value: "\\g",
kind: String,
triple_quoted: false,
},
43..47,
),
(
String {
value: "raw\\'",
kind: RawString,
triple_quoted: false,
},
48..56,
),
(
String {
value: "\\420",
kind: String,
triple_quoted: false,
},
57..63,
),
(
String {
value: "\\200\\0a",
kind: String,
triple_quoted: false,
},
64..73,
),
(
Newline,
73..73,
),
]

View file

@ -1,12 +1,18 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: string_continuation_with_eol(MAC_EOL)
---
[
String {
value: "abc\\\rdef",
kind: String,
triple_quoted: false,
},
Newline,
(
String {
value: "abc\\\rdef",
kind: String,
triple_quoted: false,
},
0..10,
),
(
Newline,
10..10,
),
]

View file

@ -1,12 +1,18 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: string_continuation_with_eol(UNIX_EOL)
---
[
String {
value: "abc\\\ndef",
kind: String,
triple_quoted: false,
},
Newline,
(
String {
value: "abc\\\ndef",
kind: String,
triple_quoted: false,
},
0..10,
),
(
Newline,
10..10,
),
]

View file

@ -1,12 +1,18 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: string_continuation_with_eol(WINDOWS_EOL)
---
[
String {
value: "abc\\\r\ndef",
kind: String,
triple_quoted: false,
},
Newline,
(
String {
value: "abc\\\r\ndef",
kind: String,
triple_quoted: false,
},
0..11,
),
(
Newline,
11..11,
),
]

View file

@ -1,12 +1,18 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: triple_quoted_eol(MAC_EOL)
---
[
String {
value: "\r test string\r ",
kind: String,
triple_quoted: true,
},
Newline,
(
String {
value: "\r test string\r ",
kind: String,
triple_quoted: true,
},
0..21,
),
(
Newline,
21..21,
),
]

View file

@ -1,12 +1,18 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: triple_quoted_eol(UNIX_EOL)
---
[
String {
value: "\n test string\n ",
kind: String,
triple_quoted: true,
},
Newline,
(
String {
value: "\n test string\n ",
kind: String,
triple_quoted: true,
},
0..21,
),
(
Newline,
21..21,
),
]

View file

@ -1,12 +1,18 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: lex_source(&source)
expression: triple_quoted_eol(WINDOWS_EOL)
---
[
String {
value: "\r\n test string\r\n ",
kind: String,
triple_quoted: true,
},
Newline,
(
String {
value: "\r\n test string\r\n ",
kind: String,
triple_quoted: true,
},
0..23,
),
(
Newline,
23..23,
),
]