mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-24 13:33:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Rename Magic* to IpyEscape* (#6395)

Browse source 
## Summary

This PR renames the `MagicCommand` token to `IpyEscapeCommand` token and
`MagicKind` to `IpyEscapeKind` type to better reflect the purpose of the
token and type. Similarly, it renames the AST nodes from `LineMagic` to
`IpyEscapeCommand` prefixed with `Stmt`/`Expr` wherever necessary.

It also makes renames from using `jupyter_magic` to
`ipython_escape_commands` in various function names.

The mode value is still `Mode::Jupyter` because the escape commands are
part of the IPython syntax but the lexing/parsing is done for a Jupyter
notebook.

### Motivation behind the rename:
* IPython codebase defines it as "EscapeCommand" / "Escape Sequences":
* Escape Sequences:
292e3a2345/IPython/core/inputtransformer2.py (L329-L333)
* Escape command:
292e3a2345/IPython/core/inputtransformer2.py (L410-L411)
* The word "magic" is used mainly for the actual magic commands i.e.,
the ones starting with `%`/`%%`
(https://ipython.readthedocs.io/en/stable/interactive/reference.html#magic-command-system).
So, this avoids any confusion between the Magic token (`%`, `%%`) and
the escape command itself.
## Test Plan

* `cargo test` to make sure all renames are done correctly.
* `grep` for `jupyter_escape`/`magic` to make sure all renames are done
correctly.
This commit is contained in:
Dhruv Manilawala 2023-08-09 18:58:18 +05:30 • committed by GitHub
parent 3bf1c66cda
commit 6a64f2289b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 949 additions and 946 deletions
Download patch file Download diff file Expand all files Collapse all files

265
crates/ruff_python_parser/src/lexer.rs
View file

@ -34,7 +34,7 @@ use std::{char, cmp::Ordering, str::FromStr};
use num_bigint::BigInt;
use num_traits::{Num, Zero};
use ruff_python_ast::MagicKind;
use ruff_python_ast::IpyEscapeKind;
use ruff_text_size::{TextLen, TextRange, TextSize};
use unic_emoji_char::is_emoji_presentation;
use unic_ucd_ident::{is_xid_continue, is_xid_start};
@ -398,8 +398,8 @@ impl<'source> Lexer<'source> {
Tok::Comment(self.token_text().to_string())
}
/// Lex a single magic command.
fn lex_magic_command(&mut self, kind: MagicKind) -> Tok {
/// Lex a single IPython escape command.
fn lex_ipython_escape_command(&mut self, escape_kind: IpyEscapeKind) -> Tok {
let mut value = String::new();
loop {
@ -457,7 +457,7 @@ impl<'source> Lexer<'source> {
// Now, the whitespace and empty value check also makes sure that an empty
// command (e.g. `%?` or `? ??`, no value after/between the escape tokens)
// is not recognized as a help end escape command. So, `%?` and `? ??` are
// `MagicKind::Magic` and `MagicKind::Help` because of the initial `%` and `??`
// `IpyEscapeKind::Magic` and `IpyEscapeKind::Help` because of the initial `%` and `??`
// tokens.
if question_count > 2
|| value.chars().last().map_or(true, is_python_whitespace)
@ -471,31 +471,34 @@ impl<'source> Lexer<'source> {
continue;
}
if kind.is_help() {
if escape_kind.is_help() {
// If we've recognize this as a help end escape command, then
// any question mark token / whitespaces at the start are not
// considered as part of the value.
//
// For example, `??foo?` is recognized as `MagicKind::Help` and
// For example, `??foo?` is recognized as `IpyEscapeKind::Help` and
// `value` is `foo` instead of `??foo`.
value = value.trim_start_matches([' ', '?']).to_string();
} else if kind.is_magic() {
} else if escape_kind.is_magic() {
// Between `%` and `?` (at the end), the `?` takes priority
// over the `%` so `%foo?` is recognized as `MagicKind::Help`
// over the `%` so `%foo?` is recognized as `IpyEscapeKind::Help`
// and `value` is `%foo` instead of `foo`. So, we need to
// insert the magic escape token at the start.
value.insert_str(0, kind.as_str());
value.insert_str(0, escape_kind.as_str());
}
let kind = match question_count {
1 => MagicKind::Help,
2 => MagicKind::Help2,
1 => IpyEscapeKind::Help,
2 => IpyEscapeKind::Help2,
_ => unreachable!("`question_count` is always 1 or 2"),
};
return Tok::MagicCommand { kind, value };
return Tok::IpyEscapeCommand { kind, value };
}
'\n' | '\r' | EOF_CHAR => {
return Tok::MagicCommand { kind, value };
return Tok::IpyEscapeCommand {
kind: escape_kind,
value,
};
}
c => {
self.cursor.bump();
@ -763,22 +766,22 @@ impl<'source> Lexer<'source> {
&& self.state.is_after_equal()
&& self.nesting == 0 =>
{
// SAFETY: Safe because `c` has been matched against one of the possible magic command prefix
self.lex_magic_command(MagicKind::try_from(c).unwrap())
// SAFETY: Safe because `c` has been matched against one of the possible escape command token
self.lex_ipython_escape_command(IpyEscapeKind::try_from(c).unwrap())
}
c @ ('%' | '!' | '?' | '/' | ';' | ',')
if self.mode == Mode::Jupyter && self.state.is_new_logical_line() =>
{
let kind = if let Ok(kind) = MagicKind::try_from([c, self.cursor.first()]) {
let kind = if let Ok(kind) = IpyEscapeKind::try_from([c, self.cursor.first()]) {
self.cursor.bump();
kind
} else {
// SAFETY: Safe because `c` has been matched against one of the possible magic command prefix
MagicKind::try_from(c).unwrap()
// SAFETY: Safe because `c` has been matched against one of the possible escape command token
IpyEscapeKind::try_from(c).unwrap()
};
self.lex_magic_command(kind)
self.lex_ipython_escape_command(kind)
}
'?' if self.mode == Mode::Jupyter => Tok::Question,
@ -1208,7 +1211,7 @@ const fn is_python_whitespace(c: char) -> bool {
#[cfg(test)]
mod tests {
use num_bigint::BigInt;
use ruff_python_ast::MagicKind;
use ruff_python_ast::IpyEscapeKind;
use super::*;
@ -1242,15 +1245,15 @@ mod tests {
}
}
fn assert_jupyter_magic_line_continuation_with_eol(eol: &str) {
fn assert_ipython_escape_command_line_continuation_with_eol(eol: &str) {
let source = format!("%matplotlib \\{eol} --inline");
let tokens = lex_jupyter_source(&source);
assert_eq!(
tokens,
vec![
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "matplotlib --inline".to_string(),
kind: MagicKind::Magic
kind: IpyEscapeKind::Magic
},
Tok::Newline
]
@ -1258,29 +1261,29 @@ mod tests {
}
#[test]
fn test_jupyter_magic_line_continuation_unix_eol() {
assert_jupyter_magic_line_continuation_with_eol(UNIX_EOL);
fn test_ipython_escape_command_line_continuation_unix_eol() {
assert_ipython_escape_command_line_continuation_with_eol(UNIX_EOL);
}
#[test]
fn test_jupyter_magic_line_continuation_mac_eol() {
assert_jupyter_magic_line_continuation_with_eol(MAC_EOL);
fn test_ipython_escape_command_line_continuation_mac_eol() {
assert_ipython_escape_command_line_continuation_with_eol(MAC_EOL);
}
#[test]
fn test_jupyter_magic_line_continuation_windows_eol() {
assert_jupyter_magic_line_continuation_with_eol(WINDOWS_EOL);
fn test_ipython_escape_command_line_continuation_windows_eol() {
assert_ipython_escape_command_line_continuation_with_eol(WINDOWS_EOL);
}
fn assert_jupyter_magic_line_continuation_with_eol_and_eof(eol: &str) {
fn assert_ipython_escape_command_line_continuation_with_eol_and_eof(eol: &str) {
let source = format!("%matplotlib \\{eol}");
let tokens = lex_jupyter_source(&source);
assert_eq!(
tokens,
vec![
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "matplotlib ".to_string(),
kind: MagicKind::Magic
kind: IpyEscapeKind::Magic
},
Tok::Newline
]
@ -1288,70 +1291,70 @@ mod tests {
}
#[test]
fn test_jupyter_magic_line_continuation_unix_eol_and_eof() {
assert_jupyter_magic_line_continuation_with_eol_and_eof(UNIX_EOL);
fn test_ipython_escape_command_line_continuation_unix_eol_and_eof() {
assert_ipython_escape_command_line_continuation_with_eol_and_eof(UNIX_EOL);
}
#[test]
fn test_jupyter_magic_line_continuation_mac_eol_and_eof() {
assert_jupyter_magic_line_continuation_with_eol_and_eof(MAC_EOL);
fn test_ipython_escape_command_line_continuation_mac_eol_and_eof() {
assert_ipython_escape_command_line_continuation_with_eol_and_eof(MAC_EOL);
}
#[test]
fn test_jupyter_magic_line_continuation_windows_eol_and_eof() {
assert_jupyter_magic_line_continuation_with_eol_and_eof(WINDOWS_EOL);
fn test_ipython_escape_command_line_continuation_windows_eol_and_eof() {
assert_ipython_escape_command_line_continuation_with_eol_and_eof(WINDOWS_EOL);
}
#[test]
fn test_empty_jupyter_magic() {
fn test_empty_ipython_escape_command() {
let source = "%\n%%\n!\n!!\n?\n??\n/\n,\n;";
let tokens = lex_jupyter_source(source);
assert_eq!(
tokens,
vec![
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Magic2,
kind: IpyEscapeKind::Magic2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Shell,
kind: IpyEscapeKind::Shell,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::ShCap,
kind: IpyEscapeKind::ShCap,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Paren,
kind: IpyEscapeKind::Paren,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Quote,
kind: IpyEscapeKind::Quote,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: String::new(),
kind: MagicKind::Quote2,
kind: IpyEscapeKind::Quote2,
},
Tok::Newline,
]
@ -1359,7 +1362,7 @@ mod tests {
}
#[test]
fn test_jupyter_magic() {
fn test_ipython_escape_command() {
let source = r"
?foo
??foo
@ -1380,59 +1383,59 @@ mod tests {
assert_eq!(
tokens,
vec![
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "timeit a = b".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "timeit a % 3".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "matplotlib --inline".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "pwd && ls -a | sed 's/^/\\\\ /'".to_string(),
kind: MagicKind::Shell,
kind: IpyEscapeKind::Shell,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "cd /Users/foo/Library/Application\\ Support/".to_string(),
kind: MagicKind::ShCap,
kind: IpyEscapeKind::ShCap,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo 1 2".to_string(),
kind: MagicKind::Paren,
kind: IpyEscapeKind::Paren,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo 1 2".to_string(),
kind: MagicKind::Quote,
kind: IpyEscapeKind::Quote,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo 1 2".to_string(),
kind: MagicKind::Quote2,
kind: IpyEscapeKind::Quote2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "ls".to_string(),
kind: MagicKind::Shell,
kind: IpyEscapeKind::Shell,
},
Tok::Newline,
]
@ -1440,7 +1443,7 @@ mod tests {
}
#[test]
fn test_jupyter_magic_help_end() {
fn test_ipython_help_end_escape_command() {
let source = r"
?foo?
?? foo?
@ -1465,84 +1468,84 @@ mod tests {
assert_eq!(
tokens,
[
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: " foo ?".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo???".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "?foo???".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: " ?".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "??".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "%foo".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "%foo".to_string(),
kind: MagicKind::Help2,
kind: IpyEscapeKind::Help2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "foo???".to_string(),
kind: MagicKind::Magic2,
kind: IpyEscapeKind::Magic2,
},
Tok::Newline,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "pwd".to_string(),
kind: MagicKind::Help,
kind: IpyEscapeKind::Help,
},
Tok::Newline,
]
@ -1550,7 +1553,7 @@ mod tests {
}
#[test]
fn test_jupyter_magic_indentation() {
fn test_ipython_escape_command_indentation() {
let source = r"
if True:
%matplotlib \
@ -1565,9 +1568,9 @@ if True:
Tok::Colon,
Tok::Newline,
Tok::Indent,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "matplotlib --inline".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::Dedent,
@ -1576,7 +1579,7 @@ if True:
}
#[test]
fn test_jupyter_magic_assignment() {
fn test_ipython_escape_command_assignment() {
let source = r"
pwd = !pwd
foo = %timeit a = b
@ -1592,54 +1595,54 @@ baz = %matplotlib \
name: "pwd".to_string()
},
Tok::Equal,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "pwd".to_string(),
kind: MagicKind::Shell,
kind: IpyEscapeKind::Shell,
},
Tok::Newline,
Tok::Name {
name: "foo".to_string()
},
Tok::Equal,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "timeit a = b".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::Name {
name: "bar".to_string()
},
Tok::Equal,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "timeit a % 3".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
Tok::Name {
name: "baz".to_string()
},
Tok::Equal,
Tok::MagicCommand {
Tok::IpyEscapeCommand {
value: "matplotlib inline".to_string(),
kind: MagicKind::Magic,
kind: IpyEscapeKind::Magic,
},
Tok::Newline,
]
);
}
fn assert_no_jupyter_magic(tokens: &[Tok]) {
fn assert_no_ipython_escape_command(tokens: &[Tok]) {
for tok in tokens {
if let Tok::MagicCommand { .. } = tok {
panic!("Unexpected magic command token: {tok:?}")
if let Tok::IpyEscapeCommand { .. } = tok {
panic!("Unexpected escape command token: {tok:?}")
}
}
}
#[test]
fn test_jupyter_magic_not_an_assignment() {
fn test_ipython_escape_command_not_an_assignment() {
let source = r"
# Other magic kinds are not valid here (can't test `foo = ?str` because '?' is not a valid token)
# Other escape kinds are not valid here (can't test `foo = ?str` because '?' is not a valid token)
foo = /func
foo = ;func
foo = ,func
@ -1650,7 +1653,7 @@ def f(arg=%timeit a = b):
pass"
.trim();
let tokens = lex_jupyter_source(source);
assert_no_jupyter_magic(&tokens);
assert_no_ipython_escape_command(&tokens);
}
#[test]

12
crates/ruff_python_parser/src/parser.rs
View file

@ -117,7 +117,7 @@ pub fn parse_expression_starts_at(
///
/// This function is the most general function to parse Python code. Based on the [`Mode`] supplied,
/// it can be used to parse a single expression, a full Python program, an interactive expression
/// or a Python program containing Jupyter magics.
/// or a Python program containing IPython escape commands.
///
/// # Example
///
@ -146,7 +146,7 @@ pub fn parse_expression_starts_at(
/// assert!(program.is_ok());
/// ```
///
/// Additionally, we can parse a Python program containing Jupyter magics:
/// Additionally, we can parse a Python program containing IPython escapes:
///
/// ```
/// use ruff_python_parser::{Mode, parse};
@ -1122,7 +1122,7 @@ class Abcd:
}
#[test]
fn test_jupyter_magic() {
fn test_ipython_escape_commands() {
let parse_ast = parse(
r#"
# Normal Python code
@ -1169,7 +1169,7 @@ def foo():
;foo 1 2
,foo 1 2
# Indented magic
# Indented escape commands
for a in range(5):
!ls
@ -1199,7 +1199,7 @@ foo.bar[0].baz[2].egg??
}
#[test]
fn test_jupyter_magic_parse_error() {
fn test_ipython_escape_command_parse_error() {
let source = r#"
a = 1
%timeit a == 1
@ -1209,7 +1209,7 @@ a = 1
let parse_err = parse_tokens(lxr, Mode::Module, "<test>").unwrap_err();
assert_eq!(
parse_err.to_string(),
"line magics are only allowed in Jupyter mode at byte offset 6".to_string()
"IPython escape commands are only allowed in Jupyter mode at byte offset 6".to_string()
);
}
}

58
crates/ruff_python_parser/src/python.lalrpop
View file

@ -5,7 +5,7 @@
use num_bigint::BigInt;
use ruff_text_size::TextSize;
use ruff_python_ast::{self as ast, Ranged, MagicKind};
use ruff_python_ast::{self as ast, Ranged, IpyEscapeKind};
use crate::{
Mode,
lexer::{LexicalError, LexicalErrorType},
@ -89,8 +89,8 @@ SmallStatement: ast::Stmt = {
NonlocalStatement,
AssertStatement,
TypeAliasStatement,
LineMagicStatement,
HelpEndLineMagic,
IpyEscapeCommandStatement,
IpyHelpEndEscapeCommandStatement,
};
PassStatement: ast::Stmt = {
@ -155,7 +155,7 @@ ExpressionStatement: ast::Stmt = {
AssignSuffix: ast::Expr = {
"=" <e:TestListOrYieldExpr> => e,
"=" <e:LineMagicExpr> => e
"=" <e:IpyEscapeCommandExpr> => e
};
TestListOrYieldExpr: ast::Expr = {
@ -323,52 +323,52 @@ AssertStatement: ast::Stmt = {
},
};
LineMagicStatement: ast::Stmt = {
<location:@L> <m:line_magic> <end_location:@R> =>? {
IpyEscapeCommandStatement: ast::Stmt = {
<location:@L> <c:ipy_escape_command> <end_location:@R> =>? {
if mode == Mode::Jupyter {
Ok(ast::Stmt::LineMagic(
ast::StmtLineMagic {
kind: m.0,
value: m.1,
Ok(ast::Stmt::IpyEscapeCommand(
ast::StmtIpyEscapeCommand {
kind: c.0,
value: c.1,
range: (location..end_location).into()
}
))
} else {
Err(LexicalError {
error: LexicalErrorType::OtherError("line magics are only allowed in Jupyter mode".to_string()),
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in Jupyter mode".to_string()),
location,
})?
}
}
}
LineMagicExpr: ast::Expr = {
<location:@L> <m:line_magic> <end_location:@R> =>? {
IpyEscapeCommandExpr: ast::Expr = {
<location:@L> <c:ipy_escape_command> <end_location:@R> =>? {
if mode == Mode::Jupyter {
// This should never occur as the lexer won't allow it.
if !matches!(m.0, MagicKind::Magic | MagicKind::Shell) {
if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) {
return Err(LexicalError {
error: LexicalErrorType::OtherError("expr line magics are only allowed for % and !".to_string()),
error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()),
location,
})?;
}
Ok(ast::Expr::LineMagic(
ast::ExprLineMagic {
kind: m.0,
value: m.1,
Ok(ast::Expr::IpyEscapeCommand(
ast::ExprIpyEscapeCommand {
kind: c.0,
value: c.1,
range: (location..end_location).into()
}
))
} else {
Err(LexicalError {
error: LexicalErrorType::OtherError("line magics are only allowed in Jupyter mode".to_string()),
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in Jupyter mode".to_string()),
location,
})?
}
}
}
HelpEndLineMagic: ast::Stmt = {
IpyHelpEndEscapeCommandStatement: ast::Stmt = {
// We are permissive than the original implementation because we would allow whitespace
// between the expression and the suffix while the IPython implementation doesn't allow it.
// For example, `foo ?` would be valid in our case but invalid from IPython.
@ -404,7 +404,7 @@ HelpEndLineMagic: ast::Stmt = {
}
Ok(())
}
if mode != Mode::Jupyter {
return Err(ParseError::User {
error: LexicalError {
@ -415,8 +415,8 @@ HelpEndLineMagic: ast::Stmt = {
}
let kind = match suffix.len() {
1 => MagicKind::Help,
2 => MagicKind::Help2,
1 => IpyEscapeKind::Help,
2 => IpyEscapeKind::Help2,
_ => {
return Err(ParseError::User {
error: LexicalError {
@ -429,9 +429,9 @@ HelpEndLineMagic: ast::Stmt = {
let mut value = String::new();
unparse_expr(&e, &mut value)?;
Ok(ast::Stmt::LineMagic(
ast::StmtLineMagic {
Ok(ast::Stmt::IpyEscapeCommand(
ast::StmtIpyEscapeCommand {
kind,
value,
range: (location..end_location).into()
@ -1900,8 +1900,8 @@ extern {
triple_quoted: <bool>
},
name => token::Tok::Name { name: <String> },
line_magic => token::Tok::MagicCommand {
kind: <MagicKind>,
ipy_escape_command => token::Tok::IpyEscapeCommand {
kind: <IpyEscapeKind>,
value: <String>
},
"\n" => token::Tok::Newline,

962
crates/ruff_python_parser/src/python.rs
View file

File diff suppressed because it is too large Load diff

174
crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__jupyter_magic.snap → crates/ruff_python_parser/src/snapshots/ruff_python_parser__parser__tests__ipython_escape_commands.snap
View file

@ -4,7 +4,7 @@ expression: parse_ast
---
Module(
ModModule {
range: 0..919,
range: 0..929,
body: [
Expr(
StmtExpr {
@ -31,92 +31,92 @@ Module(
),
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 66..73,
kind: Help2,
value: "a.foo",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 74..80,
kind: Help,
value: "a.foo",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 81..88,
kind: Help,
value: "a.foo",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 89..100,
kind: Help2,
value: "a.foo()",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 115..128,
kind: Magic,
value: "timeit a = b",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 129..147,
kind: Magic,
value: "timeit foo(b) % 3",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 148..176,
kind: Magic,
value: "alias showPath pwd && ls -a",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 177..205,
kind: Magic,
value: "timeit a = foo(b); b = 2",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 206..226,
kind: Magic,
value: "matplotlib --inline",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 227..253,
kind: Magic,
value: "matplotlib --inline",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 277..309,
kind: Shell,
value: "pwd && ls -a | sed 's/^/\\ /'",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 310..347,
kind: Shell,
value: "pwd && ls -a | sed 's/^/\\\\ /'",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 348..393,
kind: ShCap,
value: "cd /Users/foo/Library/Application\\ Support/",
@ -176,22 +176,22 @@ Module(
],
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 656..664,
kind: Paren,
value: "foo 1 2",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 665..673,
kind: Quote2,
value: "foo 1 2",
},
),
LineMagic(
StmtLineMagic {
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 674..682,
kind: Quote,
value: "foo 1 2",
@ -199,31 +199,31 @@ Module(
),
For(
StmtFor {
range: 701..727,
range: 711..737,
is_async: false,
target: Name(
ExprName {
range: 705..706,
range: 715..716,
id: "a",
ctx: Store,
},
),
iter: Call(
ExprCall {
range: 710..718,
range: 720..728,
func: Name(
ExprName {
range: 710..715,
range: 720..725,
id: "range",
ctx: Load,
},
),
arguments: Arguments {
range: 715..718,
range: 725..728,
args: [
Constant(
ExprConstant {
range: 716..717,
range: 726..727,
value: Int(
5,
),
@ -236,9 +236,9 @@ Module(
},
),
body: [
LineMagic(
StmtLineMagic {
range: 724..727,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 734..737,
kind: Shell,
value: "ls",
},
@ -249,19 +249,19 @@ Module(
),
Assign(
StmtAssign {
range: 729..738,
range: 739..748,
targets: [
Name(
ExprName {
range: 729..731,
range: 739..741,
id: "p1",
ctx: Store,
},
),
],
value: LineMagic(
ExprLineMagic {
range: 734..738,
value: IpyEscapeCommand(
ExprIpyEscapeCommand {
range: 744..748,
kind: Shell,
value: "pwd",
},
@ -270,25 +270,25 @@ Module(
),
AnnAssign(
StmtAnnAssign {
range: 739..753,
range: 749..763,
target: Name(
ExprName {
range: 739..741,
range: 749..751,
id: "p2",
ctx: Store,
},
),
annotation: Name(
ExprName {
range: 743..746,
range: 753..756,
id: "str",
ctx: Load,
},
),
value: Some(
LineMagic(
ExprLineMagic {
range: 749..753,
IpyEscapeCommand(
ExprIpyEscapeCommand {
range: 759..763,
kind: Shell,
value: "pwd",
},
@ -299,98 +299,98 @@ Module(
),
Assign(
StmtAssign {
range: 754..774,
range: 764..784,
targets: [
Name(
ExprName {
range: 754..757,
range: 764..767,
id: "foo",
ctx: Store,
},
),
],
value: LineMagic(
ExprLineMagic {
range: 760..774,
value: IpyEscapeCommand(
ExprIpyEscapeCommand {
range: 770..784,
kind: Magic,
value: "foo bar",
},
),
},
),
LineMagic(
StmtLineMagic {
range: 776..781,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 786..791,
kind: Magic,
value: " foo",
},
),
Assign(
StmtAssign {
range: 782..803,
range: 792..813,
targets: [
Name(
ExprName {
range: 782..785,
range: 792..795,
id: "foo",
ctx: Store,
},
),
],
value: LineMagic(
ExprLineMagic {
range: 788..803,
value: IpyEscapeCommand(
ExprIpyEscapeCommand {
range: 798..813,
kind: Magic,
value: "foo # comment",
},
),
},
),
LineMagic(
StmtLineMagic {
range: 828..832,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 838..842,
kind: Help,
value: "foo",
},
),
LineMagic(
StmtLineMagic {
range: 833..842,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 843..852,
kind: Help2,
value: "foo.bar",
},
),
LineMagic(
StmtLineMagic {
range: 843..855,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 853..865,
kind: Help,
value: "foo.bar.baz",
},
),
LineMagic(
StmtLineMagic {
range: 856..864,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 866..874,
kind: Help2,
value: "foo[0]",
},
),
LineMagic(
StmtLineMagic {
range: 865..875,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 875..885,
kind: Help,
value: "foo[0][1]",
},
),
LineMagic(
StmtLineMagic {
range: 876..895,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 886..905,
kind: Help2,
value: "foo.bar[0].baz[1]",
},
),
LineMagic(
StmtLineMagic {
range: 896..919,
IpyEscapeCommand(
StmtIpyEscapeCommand {
range: 906..929,
kind: Help2,
value: "foo.bar[0].baz[2].egg",
},

18
crates/ruff_python_parser/src/token.rs
View file

@ -6,7 +6,7 @@
//! [CPython source]: https://github.com/python/cpython/blob/dfc2e065a2e71011017077e549cd2f9bf4944c54/Include/internal/pycore_token.h;
use crate::Mode;
use num_bigint::BigInt;
use ruff_python_ast::MagicKind;
use ruff_python_ast::IpyEscapeKind;
use ruff_text_size::TextSize;
use std::fmt;
@ -44,13 +44,13 @@ pub enum Tok {
/// Whether the string is triple quoted.
triple_quoted: bool,
},
/// Token value for a Jupyter magic commands. These are filtered out of the token stream
/// prior to parsing when the mode is [`Mode::Jupyter`].
MagicCommand {
/// Token value for IPython escape commands. These are recognized by the lexer
/// only when the mode is [`Mode::Jupyter`].
IpyEscapeCommand {
/// The magic command value.
value: String,
/// The kind of magic command.
kind: MagicKind,
kind: IpyEscapeKind,
},
/// Token value for a comment. These are filtered out of the token stream prior to parsing.
Comment(String),
@ -234,7 +234,7 @@ impl fmt::Display for Tok {
let quotes = "\"".repeat(if *triple_quoted { 3 } else { 1 });
write!(f, "{kind}{quotes}{value}{quotes}")
}
MagicCommand { kind, value } => write!(f, "{kind}{value}"),
IpyEscapeCommand { kind, value } => write!(f, "{kind}{value}"),
Newline => f.write_str("Newline"),
NonLogicalNewline => f.write_str("NonLogicalNewline"),
Indent => f.write_str("Indent"),
@ -450,8 +450,8 @@ pub enum TokenKind {
Complex,
/// Token value for a string.
String,
/// Token value for a Jupyter magic command.
MagicCommand,
/// Token value for a IPython escape command.
EscapeCommand,
/// Token value for a comment. These are filtered out of the token stream prior to parsing.
Comment,
/// Token value for a newline.
@ -781,7 +781,7 @@ impl TokenKind {
Tok::Float { .. } => TokenKind::Float,
Tok::Complex { .. } => TokenKind::Complex,
Tok::String { .. } => TokenKind::String,
Tok::MagicCommand { .. } => TokenKind::MagicCommand,
Tok::IpyEscapeCommand { .. } => TokenKind::EscapeCommand,
Tok::Comment(_) => TokenKind::Comment,
Tok::Newline => TokenKind::Newline,
Tok::NonLogicalNewline => TokenKind::NonLogicalNewline,