mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-08-04 02:38:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Reduce Result<Tok, LexicalError> size by using Box<str> instead of String (#9885)

Browse source
This commit is contained in:
Micha Reiser 2024-02-08 21:36:22 +01:00 • committed by GitHub
parent 9027169125
commit fe7d965334
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 454 additions and 425 deletions
Download patch file Download diff file Expand all files Collapse all files

23
crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs
View file

@ -257,15 +257,18 @@ fn elts_to_csv(elts: &[Expr], generator: Generator) -> Option<String> {
} }
let node = Expr::from(ast::StringLiteral { let node = Expr::from(ast::StringLiteral {
value: elts.iter().fold(String::new(), |mut acc, elt| { value: elts
if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt { .iter()
if !acc.is_empty() { .fold(String::new(), |mut acc, elt| {
acc.push(','); if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt {
if !acc.is_empty() {
acc.push(',');
}
acc.push_str(value.to_str());
} }
acc.push_str(value.to_str()); acc
} })
acc .into_boxed_str(),
}),
..ast::StringLiteral::default() ..ast::StringLiteral::default()
}); });
Some(generator.expr(&node)) Some(generator.expr(&node))
@ -327,7 +330,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
.iter() .iter()
.map(|name| { .map(|name| {
Expr::from(ast::StringLiteral { Expr::from(ast::StringLiteral {
value: (*name).to_string(), value: (*name).to_string().into_boxed_str(),
..ast::StringLiteral::default() ..ast::StringLiteral::default()
}) })
}) })
@ -360,7 +363,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) {
.iter() .iter()
.map(|name| { .map(|name| {
Expr::from(ast::StringLiteral { Expr::from(ast::StringLiteral {
value: (*name).to_string(), value: (*name).to_string().into_boxed_str(),
..ast::StringLiteral::default() ..ast::StringLiteral::default()
}) })
}) })

2
crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs
View file

@ -217,7 +217,7 @@ fn check_os_environ_subscript(checker: &mut Checker, expr: &Expr) {
slice.range(), slice.range(),
); );
let node = ast::StringLiteral { let node = ast::StringLiteral {
value: capital_env_var, value: capital_env_var.into_boxed_str(),
unicode: env_var.is_unicode(), unicode: env_var.is_unicode(),
..ast::StringLiteral::default() ..ast::StringLiteral::default()
}; };

3
crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs
View file

@ -72,7 +72,8 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option<Expr> {
None None
} }
}) })
.join(joiner), .join(joiner)
.into_boxed_str(),
..ast::StringLiteral::default() ..ast::StringLiteral::default()
}; };
return Some(node.into()); return Some(node.into());

2
crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs
View file

@ -74,7 +74,7 @@ pub(crate) fn invalid_escape_sequence(
let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else { let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else {
return; return;
}; };
(value.as_str(), range.start()) (&**value, range.start())
} }
Tok::String { kind, .. } => { Tok::String { kind, .. } => {
if kind.is_raw() { if kind.is_raw() {

2
crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs
View file

@ -110,7 +110,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix {
.generator() .generator()
.expr(&Expr::StringLiteral(ast::ExprStringLiteral { .expr(&Expr::StringLiteral(ast::ExprStringLiteral {
value: ast::StringLiteralValue::single(ast::StringLiteral { value: ast::StringLiteralValue::single(ast::StringLiteral {
value: "locale".to_string(), value: "locale".to_string().into_boxed_str(),
unicode: false, unicode: false,
range: TextRange::default(), range: TextRange::default(),
}), }),

2
crates/ruff_linter/src/rules/pyupgrade/fixes.rs
View file

@ -21,7 +21,7 @@ pub(crate) fn remove_import_members(contents: &str, members: &[&str]) -> String
let last_range = names.last_mut().unwrap(); let last_range = names.last_mut().unwrap();
*last_range = TextRange::new(last_range.start(), range.end()); *last_range = TextRange::new(last_range.start(), range.end());
} else { } else {
if members.contains(&name.as_str()) { if members.contains(&&**name) {
removal_indices.push(names.len()); removal_indices.push(names.len());
} }
names.push(range); names.push(range);

16
crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs
View file

@ -559,14 +559,14 @@ fn collect_string_sequence_lines(
/// `self` and produces the classification for the line. /// `self` and produces the classification for the line.
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct LineState { struct LineState {
first_item_in_line: Option<(String, TextRange)>, first_item_in_line: Option<(Box<str>, TextRange)>,
following_items_in_line: Vec<(String, TextRange)>, following_items_in_line: Vec<(Box<str>, TextRange)>,
comment_range_start: Option<TextSize>, comment_range_start: Option<TextSize>,
comment_in_line: Option<TextRange>, comment_in_line: Option<TextRange>,
} }
impl LineState { impl LineState {
fn visit_string_token(&mut self, token_value: String, token_range: TextRange) { fn visit_string_token(&mut self, token_value: Box<str>, token_range: TextRange) {
if self.first_item_in_line.is_none() { if self.first_item_in_line.is_none() {
self.first_item_in_line = Some((token_value, token_range)); self.first_item_in_line = Some((token_value, token_range));
} else { } else {
@ -631,8 +631,8 @@ struct LineWithItems {
// For elements in the list, we keep track of the value of the // For elements in the list, we keep track of the value of the
// value of the element as well as the source-code range of the element. // value of the element as well as the source-code range of the element.
// (We need to know the actual value so that we can sort the items.) // (We need to know the actual value so that we can sort the items.)
first_item: (String, TextRange), first_item: (Box<str>, TextRange),
following_items: Vec<(String, TextRange)>, following_items: Vec<(Box<str>, TextRange)>,
// For comments, we only need to keep track of the source-code range. // For comments, we only need to keep track of the source-code range.
trailing_comment_range: Option<TextRange>, trailing_comment_range: Option<TextRange>,
} }
@ -753,7 +753,7 @@ fn collect_string_sequence_items(
/// source-code range of `"a"`. /// source-code range of `"a"`.
#[derive(Debug)] #[derive(Debug)]
struct StringSequenceItem { struct StringSequenceItem {
value: String, value: Box<str>,
preceding_comment_ranges: Vec<TextRange>, preceding_comment_ranges: Vec<TextRange>,
element_range: TextRange, element_range: TextRange,
// total_range incorporates the ranges of preceding comments // total_range incorporates the ranges of preceding comments
@ -766,7 +766,7 @@ struct StringSequenceItem {
impl StringSequenceItem { impl StringSequenceItem {
fn new( fn new(
value: String, value: Box<str>,
preceding_comment_ranges: Vec<TextRange>, preceding_comment_ranges: Vec<TextRange>,
element_range: TextRange, element_range: TextRange,
end_of_line_comments: Option<TextRange>, end_of_line_comments: Option<TextRange>,
@ -787,7 +787,7 @@ impl StringSequenceItem {
} }
} }
fn with_no_comments(value: String, element_range: TextRange) -> Self { fn with_no_comments(value: Box<str>, element_range: TextRange) -> Self {
Self::new(value, vec![], element_range, None) Self::new(value, vec![], element_range, None)
} }
} }

12
crates/ruff_python_ast/src/comparable.rs
View file

@ -631,7 +631,7 @@ pub struct ComparableStringLiteral<'a> {
impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> { impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> {
fn from(string_literal: &'a ast::StringLiteral) -> Self { fn from(string_literal: &'a ast::StringLiteral) -> Self {
Self { Self {
value: string_literal.value.as_str(), value: &string_literal.value,
} }
} }
} }
@ -1089,10 +1089,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
kind, kind,
value, value,
range: _, range: _,
}) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }),
kind: *kind,
value: value.as_str(),
}),
} }
} }
} }
@ -1537,10 +1534,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> {
kind, kind,
value, value,
range: _, range: _,
}) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { kind: *kind, value }),
kind: *kind,
value: value.as_str(),
}),
ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr { ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr {
value: value.into(), value: value.into(),
}), }),

16
crates/ruff_python_ast/src/nodes.rs
View file

@ -160,7 +160,7 @@ pub enum Stmt {
pub struct StmtIpyEscapeCommand { pub struct StmtIpyEscapeCommand {
pub range: TextRange, pub range: TextRange,
pub kind: IpyEscapeKind, pub kind: IpyEscapeKind,
pub value: String, pub value: Box<str>,
} }
impl From<StmtIpyEscapeCommand> for Stmt { impl From<StmtIpyEscapeCommand> for Stmt {
@ -671,7 +671,7 @@ impl Expr {
pub struct ExprIpyEscapeCommand { pub struct ExprIpyEscapeCommand {
pub range: TextRange, pub range: TextRange,
pub kind: IpyEscapeKind, pub kind: IpyEscapeKind,
pub value: String, pub value: Box<str>,
} }
impl From<ExprIpyEscapeCommand> for Expr { impl From<ExprIpyEscapeCommand> for Expr {
@ -1384,7 +1384,7 @@ impl Default for StringLiteralValueInner {
#[derive(Clone, Debug, Default, PartialEq)] #[derive(Clone, Debug, Default, PartialEq)]
pub struct StringLiteral { pub struct StringLiteral {
pub range: TextRange, pub range: TextRange,
pub value: String, pub value: Box<str>,
pub unicode: bool, pub unicode: bool,
} }
@ -1398,7 +1398,7 @@ impl Deref for StringLiteral {
type Target = str; type Target = str;
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
self.value.as_str() &self.value
} }
} }
@ -1426,14 +1426,16 @@ struct ConcatenatedStringLiteral {
/// Each string literal that makes up the concatenated string. /// Each string literal that makes up the concatenated string.
strings: Vec<StringLiteral>, strings: Vec<StringLiteral>,
/// The concatenated string value. /// The concatenated string value.
value: OnceCell<String>, value: OnceCell<Box<str>>,
} }
impl ConcatenatedStringLiteral { impl ConcatenatedStringLiteral {
/// Extracts a string slice containing the entire concatenated string. /// Extracts a string slice containing the entire concatenated string.
fn to_str(&self) -> &str { fn to_str(&self) -> &str {
self.value self.value.get_or_init(|| {
.get_or_init(|| self.strings.iter().map(StringLiteral::as_str).collect()) let concatenated: String = self.strings.iter().map(StringLiteral::as_str).collect();
concatenated.into_boxed_str()
})
} }
} }

4
crates/ruff_python_formatter/src/lib.rs
View file

@ -134,8 +134,8 @@ pub fn format_module_source(
let source_type = options.source_type(); let source_type = options.source_type();
let (tokens, comment_ranges) = let (tokens, comment_ranges) =
tokens_and_ranges(source, source_type).map_err(|err| ParseError { tokens_and_ranges(source, source_type).map_err(|err| ParseError {
offset: err.location, offset: err.location(),
error: ParseErrorType::Lexical(err.error), error: ParseErrorType::Lexical(err.into_error()),
})?; })?;
let module = parse_tokens(tokens, source, source_type.as_mode())?; let module = parse_tokens(tokens, source, source_type.as_mode())?;
let formatted = format_module_ast(&module, &comment_ranges, source, options)?; let formatted = format_module_ast(&module, &comment_ranges, source, options)?;

4
crates/ruff_python_formatter/src/range.rs
View file

@ -73,8 +73,8 @@ pub fn format_range(
let (tokens, comment_ranges) = let (tokens, comment_ranges) =
tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError { tokens_and_ranges(source, options.source_type()).map_err(|err| ParseError {
offset: err.location, offset: err.location(),
error: ParseErrorType::Lexical(err.error), error: ParseErrorType::Lexical(err.into_error()),
})?; })?;
assert_valid_char_boundaries(range, source); assert_valid_char_boundaries(range, source);

12
crates/ruff_python_formatter/tests/normalizer.rs
View file

@ -95,19 +95,22 @@ impl Transformer for Normalizer {
&string_literal.value, &string_literal.value,
"<DOCTEST-CODE-SNIPPET: Removed by normalizer>\n", "<DOCTEST-CODE-SNIPPET: Removed by normalizer>\n",
) )
.into_owned(); .into_owned()
.into_boxed_str();
string_literal.value = STRIP_RST_BLOCKS string_literal.value = STRIP_RST_BLOCKS
.replace_all( .replace_all(
&string_literal.value, &string_literal.value,
"<RSTBLOCK-CODE-SNIPPET: Removed by normalizer>\n", "<RSTBLOCK-CODE-SNIPPET: Removed by normalizer>\n",
) )
.into_owned(); .into_owned()
.into_boxed_str();
string_literal.value = STRIP_MARKDOWN_BLOCKS string_literal.value = STRIP_MARKDOWN_BLOCKS
.replace_all( .replace_all(
&string_literal.value, &string_literal.value,
"<MARKDOWN-CODE-SNIPPET: Removed by normalizer>\n", "<MARKDOWN-CODE-SNIPPET: Removed by normalizer>\n",
) )
.into_owned(); .into_owned()
.into_boxed_str();
// Normalize a string by (2) stripping any leading and trailing space from each // Normalize a string by (2) stripping any leading and trailing space from each
// line, and (3) removing any blank lines from the start and end of the string. // line, and (3) removing any blank lines from the start and end of the string.
string_literal.value = string_literal string_literal.value = string_literal
@ -117,6 +120,7 @@ impl Transformer for Normalizer {
.collect::<Vec<_>>() .collect::<Vec<_>>()
.join("\n") .join("\n")
.trim() .trim()
.to_owned(); .to_owned()
.into_boxed_str();
} }
} }

64
crates/ruff_python_parser/src/function.rs
View file

@ -39,10 +39,10 @@ pub(crate) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), Lexi
let range = arg.range; let range = arg.range;
let arg_name = arg.name.as_str(); let arg_name = arg.name.as_str();
if !all_arg_names.insert(arg_name) { if !all_arg_names.insert(arg_name) {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()), LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()),
location: range.start(), range.start(),
}); ));
} }
} }
@ -64,10 +64,10 @@ pub(crate) fn validate_pos_params(
.skip_while(|arg| arg.default.is_some()) // and then args with default .skip_while(|arg| arg.default.is_some()) // and then args with default
.next(); // there must not be any more args without default .next(); // there must not be any more args without default
if let Some(invalid) = first_invalid { if let Some(invalid) = first_invalid {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::DefaultArgumentError, LexicalErrorType::DefaultArgumentError,
location: invalid.parameter.start(), invalid.parameter.start(),
}); ));
} }
Ok(()) Ok(())
} }
@ -94,12 +94,12 @@ pub(crate) fn parse_arguments(
// Check for duplicate keyword arguments in the call. // Check for duplicate keyword arguments in the call.
if let Some(keyword_name) = &name { if let Some(keyword_name) = &name {
if !keyword_names.insert(keyword_name.to_string()) { if !keyword_names.insert(keyword_name.to_string()) {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::DuplicateKeywordArgumentError( LexicalErrorType::DuplicateKeywordArgumentError(
keyword_name.to_string(), keyword_name.to_string().into_boxed_str(),
), ),
location: start, start,
}); ));
} }
} else { } else {
double_starred = true; double_starred = true;
@ -113,17 +113,17 @@ pub(crate) fn parse_arguments(
} else { } else {
// Positional arguments mustn't follow keyword arguments. // Positional arguments mustn't follow keyword arguments.
if !keywords.is_empty() && !is_starred(&value) { if !keywords.is_empty() && !is_starred(&value) {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::PositionalArgumentError, LexicalErrorType::PositionalArgumentError,
location: value.start(), value.start(),
}); ));
// Allow starred arguments after keyword arguments but // Allow starred arguments after keyword arguments but
// not after double-starred arguments. // not after double-starred arguments.
} else if double_starred { } else if double_starred {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::UnpackedArgumentError, LexicalErrorType::UnpackedArgumentError,
location: value.start(), value.start(),
}); ));
} }
args.push(value); args.push(value);
@ -202,22 +202,22 @@ mod tests {
function_and_lambda_error! { function_and_lambda_error! {
// Check definitions // Check definitions
test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError, test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError,
test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError, test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError,
// Check some calls. // Check some calls.
test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError, test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError,
test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError, test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError,
test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()), test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string().into_boxed_str()),
} }
} }

2
crates/ruff_python_parser/src/invalid.rs
View file

@ -39,7 +39,7 @@ pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> {
let err = |location: TextSize| -> LexicalError { let err = |location: TextSize| -> LexicalError {
let error = LexicalErrorType::AssignmentError; let error = LexicalErrorType::AssignmentError;
LexicalError { error, location } LexicalError::new(error, location)
}; };
match *target { match *target {
BoolOp(ref e) => Err(err(e.range.start())), BoolOp(ref e) => Err(err(e.range.start())),

244
crates/ruff_python_parser/src/lexer.rs
View file

@ -107,10 +107,10 @@ where
fn next(&mut self) -> Option<Self::Item> { fn next(&mut self) -> Option<Self::Item> {
let result = match self.inner.next()? { let result = match self.inner.next()? {
Ok((tok, range)) => Ok((tok, range + self.start_offset)), Ok((tok, range)) => Ok((tok, range + self.start_offset)),
Err(error) => Err(LexicalError { Err(error) => {
location: error.location + self.start_offset, let location = error.location() + self.start_offset;
..error Err(LexicalError::new(error.into_error(), location))
}), }
}; };
Some(result) Some(result)
@ -241,7 +241,7 @@ impl<'source> Lexer<'source> {
"yield" => Tok::Yield, "yield" => Tok::Yield,
_ => { _ => {
return Ok(Tok::Name { return Ok(Tok::Name {
name: text.to_string(), name: text.to_string().into_boxed_str(),
}) })
} }
}; };
@ -284,10 +284,10 @@ impl<'source> Lexer<'source> {
let value = match Int::from_str_radix(number.as_str(), radix.as_u32(), token) { let value = match Int::from_str_radix(number.as_str(), radix.as_u32(), token) {
Ok(int) => int, Ok(int) => int,
Err(err) => { Err(err) => {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError(format!("{err:?}")), LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()),
location: self.token_range().start(), self.token_range().start(),
}); ));
} }
}; };
Ok(Tok::Int { value }) Ok(Tok::Int { value })
@ -309,10 +309,10 @@ impl<'source> Lexer<'source> {
number.push('.'); number.push('.');
if self.cursor.eat_char('_') { if self.cursor.eat_char('_') {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()), LexicalErrorType::OtherError("Invalid Syntax".to_string().into_boxed_str()),
location: self.offset() - TextSize::new(1), self.offset() - TextSize::new(1),
}); ));
} }
self.radix_run(&mut number, Radix::Decimal); self.radix_run(&mut number, Radix::Decimal);
@ -340,9 +340,13 @@ impl<'source> Lexer<'source> {
if is_float { if is_float {
// Improvement: Use `Cow` instead of pushing to value text // Improvement: Use `Cow` instead of pushing to value text
let value = f64::from_str(number.as_str()).map_err(|_| LexicalError { let value = f64::from_str(number.as_str()).map_err(|_| {
error: LexicalErrorType::OtherError("Invalid decimal literal".to_owned()), LexicalError::new(
location: self.token_start(), LexicalErrorType::OtherError(
"Invalid decimal literal".to_string().into_boxed_str(),
),
self.token_start(),
)
})?; })?;
// Parse trailing 'j': // Parse trailing 'j':
@ -364,18 +368,20 @@ impl<'source> Lexer<'source> {
Ok(value) => { Ok(value) => {
if start_is_zero && value.as_u8() != Some(0) { if start_is_zero && value.as_u8() != Some(0) {
// Leading zeros in decimal integer literals are not permitted. // Leading zeros in decimal integer literals are not permitted.
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("Invalid Token".to_owned()), LexicalErrorType::OtherError(
location: self.token_range().start(), "Invalid Token".to_string().into_boxed_str(),
}); ),
self.token_range().start(),
));
} }
value value
} }
Err(err) => { Err(err) => {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError(format!("{err:?}")), LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()),
location: self.token_range().start(), self.token_range().start(),
}) ))
} }
}; };
Ok(Tok::Int { value }) Ok(Tok::Int { value })
@ -411,7 +417,7 @@ impl<'source> Lexer<'source> {
let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len()); let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len());
self.cursor.skip_bytes(offset); self.cursor.skip_bytes(offset);
Tok::Comment(self.token_text().to_string()) Tok::Comment(self.token_text().to_string().into_boxed_str())
} }
/// Lex a single IPython escape command. /// Lex a single IPython escape command.
@ -508,12 +514,15 @@ impl<'source> Lexer<'source> {
2 => IpyEscapeKind::Help2, 2 => IpyEscapeKind::Help2,
_ => unreachable!("`question_count` is always 1 or 2"), _ => unreachable!("`question_count` is always 1 or 2"),
}; };
return Tok::IpyEscapeCommand { kind, value }; return Tok::IpyEscapeCommand {
kind,
value: value.into_boxed_str(),
};
} }
'\n' | '\r' | EOF_CHAR => { '\n' | '\r' | EOF_CHAR => {
return Tok::IpyEscapeCommand { return Tok::IpyEscapeCommand {
kind: escape_kind, kind: escape_kind,
value, value: value.into_boxed_str(),
}; };
} }
c => { c => {
@ -584,10 +593,10 @@ impl<'source> Lexer<'source> {
} else { } else {
FStringErrorType::UnterminatedString FStringErrorType::UnterminatedString
}; };
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(error), LexicalErrorType::FStringError(error),
location: self.offset(), self.offset(),
}); ));
} }
'\n' | '\r' if !fstring.is_triple_quoted() => { '\n' | '\r' if !fstring.is_triple_quoted() => {
// If we encounter a newline while we're in a format spec, then // If we encounter a newline while we're in a format spec, then
@ -597,10 +606,10 @@ impl<'source> Lexer<'source> {
if in_format_spec { if in_format_spec {
break; break;
} }
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::UnterminatedString), LexicalErrorType::FStringError(FStringErrorType::UnterminatedString),
location: self.offset(), self.offset(),
}); ));
} }
'\\' => { '\\' => {
self.cursor.bump(); // '\' self.cursor.bump(); // '\'
@ -673,7 +682,7 @@ impl<'source> Lexer<'source> {
normalized normalized
}; };
Ok(Some(Tok::FStringMiddle { Ok(Some(Tok::FStringMiddle {
value, value: value.into_boxed_str(),
is_raw: fstring.is_raw_string(), is_raw: fstring.is_raw_string(),
triple_quoted: fstring.is_triple_quoted(), triple_quoted: fstring.is_triple_quoted(),
})) }))
@ -705,18 +714,16 @@ impl<'source> Lexer<'source> {
if fstring.quote_char() == quote if fstring.quote_char() == quote
&& fstring.is_triple_quoted() == triple_quoted && fstring.is_triple_quoted() == triple_quoted
{ {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError( LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace),
FStringErrorType::UnclosedLbrace, self.cursor.text_len(),
), ));
location: self.cursor.text_len(),
});
} }
} }
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::Eof, LexicalErrorType::Eof,
location: self.cursor.text_len(), self.cursor.text_len(),
}); ));
}; };
// Rare case: if there are an odd number of backslashes before the quote, then // Rare case: if there are an odd number of backslashes before the quote, then
@ -756,18 +763,16 @@ impl<'source> Lexer<'source> {
if fstring.quote_char() == quote if fstring.quote_char() == quote
&& fstring.is_triple_quoted() == triple_quoted && fstring.is_triple_quoted() == triple_quoted
{ {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError( LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace),
FStringErrorType::UnclosedLbrace, self.offset(),
), ));
location: self.offset(),
});
} }
} }
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::StringError, LexicalErrorType::StringError,
location: self.offset(), self.offset(),
}); ));
}; };
// Rare case: if there are an odd number of backslashes before the quote, then // Rare case: if there are an odd number of backslashes before the quote, then
@ -797,20 +802,22 @@ impl<'source> Lexer<'source> {
// matches with f-strings quotes and if it is, then this must be a // matches with f-strings quotes and if it is, then this must be a
// missing '}' token so raise the proper error. // missing '}' token so raise the proper error.
if fstring.quote_char() == quote && !fstring.is_triple_quoted() { if fstring.quote_char() == quote && !fstring.is_triple_quoted() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError( LexicalErrorType::FStringError(
FStringErrorType::UnclosedLbrace, FStringErrorType::UnclosedLbrace,
), ),
location: self.offset() - TextSize::new(1), self.offset() - TextSize::new(1),
}); ));
} }
} }
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError( LexicalErrorType::OtherError(
"EOL while scanning string literal".to_owned(), "EOL while scanning string literal"
.to_string()
.into_boxed_str(),
), ),
location: self.offset() - TextSize::new(1), self.offset() - TextSize::new(1),
}); ));
} }
Some(ch) if ch == quote => { Some(ch) if ch == quote => {
break self.offset() - TextSize::new(1); break self.offset() - TextSize::new(1);
@ -821,7 +828,9 @@ impl<'source> Lexer<'source> {
}; };
Ok(Tok::String { Ok(Tok::String {
value: self.source[TextRange::new(value_start, value_end)].to_string(), value: self.source[TextRange::new(value_start, value_end)]
.to_string()
.into_boxed_str(),
kind, kind,
triple_quoted, triple_quoted,
}) })
@ -889,10 +898,10 @@ impl<'source> Lexer<'source> {
Ok((identifier, self.token_range())) Ok((identifier, self.token_range()))
} else { } else {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::UnrecognizedToken { tok: c }, LexicalErrorType::UnrecognizedToken { tok: c },
location: self.token_start(), self.token_start(),
}) ))
} }
} else { } else {
// Reached the end of the file. Emit a trailing newline token if not at the beginning of a logical line, // Reached the end of the file. Emit a trailing newline token if not at the beginning of a logical line,
@ -915,15 +924,12 @@ impl<'source> Lexer<'source> {
if self.cursor.eat_char('\r') { if self.cursor.eat_char('\r') {
self.cursor.eat_char('\n'); self.cursor.eat_char('\n');
} else if self.cursor.is_eof() { } else if self.cursor.is_eof() {
return Err(LexicalError { return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start()));
error: LexicalErrorType::Eof,
location: self.token_start(),
});
} else if !self.cursor.eat_char('\n') { } else if !self.cursor.eat_char('\n') {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::LineContinuationError, LexicalErrorType::LineContinuationError,
location: self.token_start(), self.token_start(),
}); ));
} }
} }
// Form feed // Form feed
@ -956,15 +962,12 @@ impl<'source> Lexer<'source> {
if self.cursor.eat_char('\r') { if self.cursor.eat_char('\r') {
self.cursor.eat_char('\n'); self.cursor.eat_char('\n');
} else if self.cursor.is_eof() { } else if self.cursor.is_eof() {
return Err(LexicalError { return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start()));
error: LexicalErrorType::Eof,
location: self.token_start(),
});
} else if !self.cursor.eat_char('\n') { } else if !self.cursor.eat_char('\n') {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::LineContinuationError, LexicalErrorType::LineContinuationError,
location: self.token_start(), self.token_start(),
}); ));
} }
indentation = Indentation::root(); indentation = Indentation::root();
} }
@ -1015,10 +1018,10 @@ impl<'source> Lexer<'source> {
Some((Tok::Indent, self.token_range())) Some((Tok::Indent, self.token_range()))
} }
Err(_) => { Err(_) => {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::IndentationError, LexicalErrorType::IndentationError,
location: self.offset(), self.offset(),
}); ));
} }
}; };
@ -1031,10 +1034,7 @@ impl<'source> Lexer<'source> {
if self.nesting > 0 { if self.nesting > 0 {
// Reset the nesting to avoid going into infinite loop. // Reset the nesting to avoid going into infinite loop.
self.nesting = 0; self.nesting = 0;
return Err(LexicalError { return Err(LexicalError::new(LexicalErrorType::Eof, self.offset()));
error: LexicalErrorType::Eof,
location: self.offset(),
});
} }
// Next, insert a trailing newline, if required. // Next, insert a trailing newline, if required.
@ -1199,10 +1199,10 @@ impl<'source> Lexer<'source> {
'}' => { '}' => {
if let Some(fstring) = self.fstrings.current_mut() { if let Some(fstring) = self.fstrings.current_mut() {
if fstring.nesting() == self.nesting { if fstring.nesting() == self.nesting {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::SingleRbrace), LexicalErrorType::FStringError(FStringErrorType::SingleRbrace),
location: self.token_start(), self.token_start(),
}); ));
} }
fstring.try_end_format_spec(self.nesting); fstring.try_end_format_spec(self.nesting);
} }
@ -1293,10 +1293,10 @@ impl<'source> Lexer<'source> {
_ => { _ => {
self.state = State::Other; self.state = State::Other;
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::UnrecognizedToken { tok: c }, LexicalErrorType::UnrecognizedToken { tok: c },
location: self.token_start(), self.token_start(),
}); ));
} }
}; };
@ -1357,9 +1357,9 @@ impl FusedIterator for Lexer<'_> {}
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct LexicalError { pub struct LexicalError {
/// The type of error that occurred. /// The type of error that occurred.
pub error: LexicalErrorType, error: LexicalErrorType,
/// The location of the error. /// The location of the error.
pub location: TextSize, location: TextSize,
} }
impl LexicalError { impl LexicalError {
@ -1367,19 +1367,31 @@ impl LexicalError {
pub fn new(error: LexicalErrorType, location: TextSize) -> Self { pub fn new(error: LexicalErrorType, location: TextSize) -> Self {
Self { error, location } Self { error, location }
} }
pub fn error(&self) -> &LexicalErrorType {
&self.error
}
pub fn into_error(self) -> LexicalErrorType {
self.error
}
pub fn location(&self) -> TextSize {
self.location
}
} }
impl std::ops::Deref for LexicalError { impl std::ops::Deref for LexicalError {
type Target = LexicalErrorType; type Target = LexicalErrorType;
fn deref(&self) -> &Self::Target { fn deref(&self) -> &Self::Target {
&self.error self.error()
} }
} }
impl std::error::Error for LexicalError { impl std::error::Error for LexicalError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error) Some(self.error())
} }
} }
@ -1388,8 +1400,8 @@ impl std::fmt::Display for LexicalError {
write!( write!(
f, f,
"{} at byte offset {}", "{} at byte offset {}",
&self.error, self.error(),
u32::from(self.location) u32::from(self.location())
) )
} }
} }
@ -1397,6 +1409,9 @@ impl std::fmt::Display for LexicalError {
/// Represents the different types of errors that can occur during lexing. /// Represents the different types of errors that can occur during lexing.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum LexicalErrorType { pub enum LexicalErrorType {
/// A duplicate argument was found in a function definition.
DuplicateArgumentError(Box<str>),
// TODO: Can probably be removed, the places it is used seem to be able // TODO: Can probably be removed, the places it is used seem to be able
// to use the `UnicodeError` variant instead. // to use the `UnicodeError` variant instead.
#[doc(hidden)] #[doc(hidden)]
@ -1414,14 +1429,13 @@ pub enum LexicalErrorType {
TabsAfterSpaces, TabsAfterSpaces,
/// A non-default argument follows a default argument. /// A non-default argument follows a default argument.
DefaultArgumentError, DefaultArgumentError,
/// A duplicate argument was found in a function definition.
DuplicateArgumentError(String),
/// A positional argument follows a keyword argument. /// A positional argument follows a keyword argument.
PositionalArgumentError, PositionalArgumentError,
/// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`. /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
UnpackedArgumentError, UnpackedArgumentError,
/// A keyword argument was repeated. /// A keyword argument was repeated.
DuplicateKeywordArgumentError(String), DuplicateKeywordArgumentError(Box<str>),
/// An unrecognized token was encountered. /// An unrecognized token was encountered.
UnrecognizedToken { tok: char }, UnrecognizedToken { tok: char },
/// An f-string error containing the [`FStringErrorType`]. /// An f-string error containing the [`FStringErrorType`].
@ -1433,7 +1447,7 @@ pub enum LexicalErrorType {
/// Occurs when a syntactically invalid assignment was encountered. /// Occurs when a syntactically invalid assignment was encountered.
AssignmentError, AssignmentError,
/// An unexpected error occurred. /// An unexpected error occurred.
OtherError(String), OtherError(Box<str>),
} }
impl std::error::Error for LexicalErrorType {} impl std::error::Error for LexicalErrorType {}
@ -2053,8 +2067,8 @@ def f(arg=%timeit a = b):
match lexed.as_slice() { match lexed.as_slice() {
[Err(error)] => { [Err(error)] => {
assert_eq!( assert_eq!(
error.error, error.error(),
LexicalErrorType::UnrecognizedToken { tok: '🐦' } &LexicalErrorType::UnrecognizedToken { tok: '🐦' }
); );
} }
result => panic!("Expected an error token but found {result:?}"), result => panic!("Expected an error token but found {result:?}"),
@ -2267,7 +2281,7 @@ f"{(lambda x:{x})}"
} }
fn lex_fstring_error(source: &str) -> FStringErrorType { fn lex_fstring_error(source: &str) -> FStringErrorType {
match lex_error(source).error { match lex_error(source).into_error() {
LexicalErrorType::FStringError(error) => error, LexicalErrorType::FStringError(error) => error,
err => panic!("Expected FStringError: {err:?}"), err => panic!("Expected FStringError: {err:?}"),
} }

8
crates/ruff_python_parser/src/parser.rs
View file

@ -285,8 +285,8 @@ fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> P
offset: token.0, offset: token.0,
}, },
LalrpopError::User { error } => ParseError { LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error), offset: error.location(),
offset: error.location, error: ParseErrorType::Lexical(error.into_error()),
}, },
LalrpopError::UnrecognizedToken { token, expected } => { LalrpopError::UnrecognizedToken { token, expected } => {
// Hacky, but it's how CPython does it. See PyParser_AddToken, // Hacky, but it's how CPython does it. See PyParser_AddToken,
@ -359,8 +359,8 @@ impl ParseErrorType {
impl From<LexicalError> for ParseError { impl From<LexicalError> for ParseError {
fn from(error: LexicalError) -> Self { fn from(error: LexicalError) -> Self {
ParseError { ParseError {
error: ParseErrorType::Lexical(error.error), offset: error.location(),
offset: error.location, error: ParseErrorType::Lexical(error.into_error()),
} }
} }
} }

108
crates/ruff_python_parser/src/python.lalrpop
View file

@ -289,7 +289,7 @@ ImportAsAlias<I>: ast::Alias = {
DottedName: ast::Identifier = { DottedName: ast::Identifier = {
<location:@L> <n:name> <end_location:@R> => ast::Identifier::new(n, (location..end_location).into()), <location:@L> <n:name> <end_location:@R> => ast::Identifier::new(n, (location..end_location).into()),
<location:@L> <n:name> <n2: ("." Identifier)+> <end_location:@R> => { <location:@L> <n:name> <n2: ("." Identifier)+> <end_location:@R> => {
let mut r = n; let mut r = String::from(n);
for x in n2 { for x in n2 {
r.push('.'); r.push('.');
r.push_str(x.1.as_str()); r.push_str(x.1.as_str());
@ -337,10 +337,10 @@ IpyEscapeCommandStatement: ast::Stmt = {
} }
)) ))
} else { } else {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location, location,
})? ))?
} }
} }
} }
@ -350,10 +350,10 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = {
if mode == Mode::Ipython { if mode == Mode::Ipython {
// This should never occur as the lexer won't allow it. // This should never occur as the lexer won't allow it.
if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) { if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()), LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()),
location, location,
})?; ))?;
} }
Ok(ast::ExprIpyEscapeCommand { Ok(ast::ExprIpyEscapeCommand {
kind: c.0, kind: c.0,
@ -361,10 +361,10 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = {
range: (location..end_location).into() range: (location..end_location).into()
}.into()) }.into())
} else { } else {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location, location,
})? ))?
} }
} }
} }
@ -381,10 +381,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
}, },
ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => { ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => {
let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else { let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()), LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()),
location: range.start(), range.start(),
}); ));
}; };
unparse_expr(value, buffer)?; unparse_expr(value, buffer)?;
buffer.push('['); buffer.push('[');
@ -397,10 +397,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
buffer.push_str(attr.as_str()); buffer.push_str(attr.as_str());
}, },
_ => { _ => {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()), LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()),
location: expr.start(), expr.start(),
}); ));
} }
} }
Ok(()) Ok(())
@ -408,10 +408,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
if mode != Mode::Ipython { if mode != Mode::Ipython {
return Err(ParseError::User { return Err(ParseError::User {
error: LexicalError { error: LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location, location,
}, ),
}); });
} }
@ -420,10 +420,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
2 => IpyEscapeKind::Help2, 2 => IpyEscapeKind::Help2,
_ => { _ => {
return Err(ParseError::User { return Err(ParseError::User {
error: LexicalError { error: LexicalError::new(
error: LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()), LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()),
location, location,
}, ),
}); });
} }
}; };
@ -434,7 +434,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
Ok(ast::Stmt::IpyEscapeCommand( Ok(ast::Stmt::IpyEscapeCommand(
ast::StmtIpyEscapeCommand { ast::StmtIpyEscapeCommand {
kind, kind,
value, value: value.into_boxed_str(),
range: (location..end_location).into() range: (location..end_location).into()
} }
)) ))
@ -561,10 +561,10 @@ Pattern: ast::Pattern = {
AsPattern: ast::Pattern = { AsPattern: ast::Pattern = {
<location:@L> <pattern:OrPattern> "as" <name:Identifier> <end_location:@R> =>? { <location:@L> <pattern:OrPattern> "as" <name:Identifier> <end_location:@R> =>? {
if name.as_str() == "_" { if name.as_str() == "_" {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()), LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()),
location, location,
})? ))?
} else { } else {
Ok(ast::Pattern::MatchAs( Ok(ast::Pattern::MatchAs(
ast::PatternMatchAs { ast::PatternMatchAs {
@ -1247,10 +1247,10 @@ DoubleStarTypedParameter: ast::Parameter = {
ParameterListStarArgs<ParameterType, StarParameterType, DoubleStarParameterType>: (Option<Box<ast::Parameter>>, Vec<ast::ParameterWithDefault>, Option<Box<ast::Parameter>>) = { ParameterListStarArgs<ParameterType, StarParameterType, DoubleStarParameterType>: (Option<Box<ast::Parameter>>, Vec<ast::ParameterWithDefault>, Option<Box<ast::Parameter>>) = {
<location:@L> "*" <va:StarParameterType?> <kwonlyargs:("," <ParameterDef<ParameterType>>)*> <kwarg:("," <KwargParameter<DoubleStarParameterType>>)?> =>? { <location:@L> "*" <va:StarParameterType?> <kwonlyargs:("," <ParameterDef<ParameterType>>)*> <kwarg:("," <KwargParameter<DoubleStarParameterType>>)?> =>? {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
location, location,
})?; ))?;
} }
let kwarg = kwarg.flatten(); let kwarg = kwarg.flatten();
@ -1364,10 +1364,10 @@ NamedExpression: crate::parser::ParenthesizedExpr = {
LambdaDef: crate::parser::ParenthesizedExpr = { LambdaDef: crate::parser::ParenthesizedExpr = {
<location:@L> "lambda" <location_args:@L> <parameters:ParameterList<UntypedParameter, StarUntypedParameter, StarUntypedParameter>?> <end_location_args:@R> ":" <fstring_middle:fstring_middle?> <body:Test<"all">> <end_location:@R> =>? { <location:@L> "lambda" <location_args:@L> <parameters:ParameterList<UntypedParameter, StarUntypedParameter, StarUntypedParameter>?> <end_location_args:@R> ":" <fstring_middle:fstring_middle?> <body:Test<"all">> <end_location:@R> =>? {
if fstring_middle.is_some() { if fstring_middle.is_some() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location, location,
})?; ))?;
} }
parameters.as_ref().map(validate_arguments).transpose()?; parameters.as_ref().map(validate_arguments).transpose()?;
@ -1630,10 +1630,10 @@ FStringMiddlePattern: ast::FStringElement = {
FStringReplacementField: ast::FStringElement = { FStringReplacementField: ast::FStringElement = {
<location:@L> "{" <value:TestListOrYieldExpr> <debug:"="?> <conversion:FStringConversion?> <format_spec:FStringFormatSpecSuffix?> "}" <end_location:@R> =>? { <location:@L> "{" <value:TestListOrYieldExpr> <debug:"="?> <conversion:FStringConversion?> <format_spec:FStringFormatSpecSuffix?> "}" <end_location:@R> =>? {
if value.expr.is_lambda_expr() && !value.is_parenthesized() { if value.expr.is_lambda_expr() && !value.is_parenthesized() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location: value.start(), value.start(),
})?; ))?;
} }
let debug_text = debug.map(|_| { let debug_text = debug.map(|_| {
let start_offset = location + "{".text_len(); let start_offset = location + "{".text_len();
@ -1677,14 +1677,14 @@ FStringFormatSpec: ast::FStringFormatSpec = {
FStringConversion: (TextSize, ast::ConversionFlag) = { FStringConversion: (TextSize, ast::ConversionFlag) = {
<location:@L> "!" <name_location:@L> <s:name> =>? { <location:@L> "!" <name_location:@L> <s:name> =>? {
let conversion = match s.as_str() { let conversion = match s.as_ref() {
"s" => ast::ConversionFlag::Str, "s" => ast::ConversionFlag::Str,
"r" => ast::ConversionFlag::Repr, "r" => ast::ConversionFlag::Repr,
"a" => ast::ConversionFlag::Ascii, "a" => ast::ConversionFlag::Ascii,
_ => Err(LexicalError { _ => Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag), LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
location: name_location, name_location,
})? ))?
}; };
Ok((location, conversion)) Ok((location, conversion))
} }
@ -1722,10 +1722,10 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
<location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? { <location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.expr.is_starred_expr() { if mid.expr.is_starred_expr() {
return Err(LexicalError{ return Err(LexicalError::new(
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
location: mid.start(), mid.start(),
})?; ))?;
} }
Ok(crate::parser::ParenthesizedExpr { Ok(crate::parser::ParenthesizedExpr {
expr: mid.into(), expr: mid.into(),
@ -1751,10 +1751,10 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
range: (location..end_location).into(), range: (location..end_location).into(),
}.into(), }.into(),
"(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? { "(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
Err(LexicalError{ Err(LexicalError::new(
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
location, location,
}.into()) ).into())
}, },
<location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => { <location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
let (keys, values) = e let (keys, values) = e
@ -2061,19 +2061,19 @@ extern {
float => token::Tok::Float { value: <f64> }, float => token::Tok::Float { value: <f64> },
complex => token::Tok::Complex { real: <f64>, imag: <f64> }, complex => token::Tok::Complex { real: <f64>, imag: <f64> },
string => token::Tok::String { string => token::Tok::String {
value: <String>, value: <Box<str>>,
kind: <StringKind>, kind: <StringKind>,
triple_quoted: <bool> triple_quoted: <bool>
}, },
fstring_middle => token::Tok::FStringMiddle { fstring_middle => token::Tok::FStringMiddle {
value: <String>, value: <Box<str>>,
is_raw: <bool>, is_raw: <bool>,
triple_quoted: <bool> triple_quoted: <bool>
}, },
name => token::Tok::Name { name: <String> }, name => token::Tok::Name { name: <Box<str>> },
ipy_escape_command => token::Tok::IpyEscapeCommand { ipy_escape_command => token::Tok::IpyEscapeCommand {
kind: <IpyEscapeKind>, kind: <IpyEscapeKind>,
value: <String> value: <Box<str>>
}, },
"\n" => token::Tok::Newline, "\n" => token::Tok::Newline,
";" => token::Tok::Semi, ";" => token::Tok::Semi,

280
crates/ruff_python_parser/src/python.rs
View file

@ -1,5 +1,5 @@
// auto-generated: "lalrpop 0.20.0" // auto-generated: "lalrpop 0.20.0"
// sha3: aa0540221d25f4eadfc9e043fb4fc631d537b672b8a96785dfec2407e0524b79 // sha3: fd05d84d3b654796ff740a7f905ec0ae8915f43f952428717735481947ab55e1
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use ruff_python_ast::{self as ast, Int, IpyEscapeKind}; use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
use crate::{ use crate::{
@ -50,11 +50,11 @@ mod __parse__Top {
Variant0(token::Tok), Variant0(token::Tok),
Variant1((f64, f64)), Variant1((f64, f64)),
Variant2(f64), Variant2(f64),
Variant3((String, bool, bool)), Variant3((Box<str>, bool, bool)),
Variant4(Int), Variant4(Int),
Variant5((IpyEscapeKind, String)), Variant5((IpyEscapeKind, Box<str>)),
Variant6(String), Variant6(Box<str>),
Variant7((String, StringKind, bool)), Variant7((Box<str>, StringKind, bool)),
Variant8(core::option::Option<token::Tok>), Variant8(core::option::Option<token::Tok>),
Variant9(Option<Box<ast::Parameter>>), Variant9(Option<Box<ast::Parameter>>),
Variant10(core::option::Option<Option<Box<ast::Parameter>>>), Variant10(core::option::Option<Option<Box<ast::Parameter>>>),
@ -151,7 +151,7 @@ mod __parse__Top {
Variant101(ast::TypeParams), Variant101(ast::TypeParams),
Variant102(core::option::Option<ast::TypeParams>), Variant102(core::option::Option<ast::TypeParams>),
Variant103(ast::UnaryOp), Variant103(ast::UnaryOp),
Variant104(core::option::Option<(String, bool, bool)>), Variant104(core::option::Option<(Box<str>, bool, bool)>),
} }
const __ACTION: &[i16] = &[ const __ACTION: &[i16] = &[
// State 0 // State 0
@ -18323,10 +18323,30 @@ mod __parse__Top {
fn __symbol_type_mismatch() -> ! { fn __symbol_type_mismatch() -> ! {
panic!("symbol type mismatch") panic!("symbol type mismatch")
} }
fn __pop_Variant7<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (Box<str>, StringKind, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant3<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (Box<str>, bool, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant5< fn __pop_Variant5<
>( >(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (IpyEscapeKind, String), TextSize) ) -> (TextSize, (IpyEscapeKind, Box<str>), TextSize)
{ {
match __symbols.pop() { match __symbols.pop() {
Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r), Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r),
@ -18373,26 +18393,6 @@ mod __parse__Top {
_ => __symbol_type_mismatch() _ => __symbol_type_mismatch()
} }
} }
fn __pop_Variant7<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (String, StringKind, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant3<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (String, bool, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant67< fn __pop_Variant67<
>( >(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -18493,6 +18493,16 @@ mod __parse__Top {
_ => __symbol_type_mismatch() _ => __symbol_type_mismatch()
} }
} }
fn __pop_Variant6<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, Box<str>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant4< fn __pop_Variant4<
>( >(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -18523,16 +18533,6 @@ mod __parse__Top {
_ => __symbol_type_mismatch() _ => __symbol_type_mismatch()
} }
} }
fn __pop_Variant6<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, String, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant69< fn __pop_Variant69<
>( >(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -19113,6 +19113,16 @@ mod __parse__Top {
_ => __symbol_type_mismatch() _ => __symbol_type_mismatch()
} }
} }
fn __pop_Variant104<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant74< fn __pop_Variant74<
>( >(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -19133,16 +19143,6 @@ mod __parse__Top {
_ => __symbol_type_mismatch() _ => __symbol_type_mismatch()
} }
} }
fn __pop_Variant104<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, core::option::Option<(String, bool, bool)>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant68< fn __pop_Variant68<
>( >(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -33541,7 +33541,7 @@ fn __action69<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, n, _): (TextSize, String, TextSize), (_, n, _): (TextSize, Box<str>, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
@ -33555,13 +33555,13 @@ fn __action70<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, n, _): (TextSize, String, TextSize), (_, n, _): (TextSize, Box<str>, TextSize),
(_, n2, _): (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), (_, n2, _): (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
{ {
let mut r = n; let mut r = String::from(n);
for x in n2 { for x in n2 {
r.push('.'); r.push('.');
r.push_str(x.1.as_str()); r.push_str(x.1.as_str());
@ -33639,7 +33639,7 @@ fn __action74<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, c, _): (TextSize, (IpyEscapeKind, String), TextSize), (_, c, _): (TextSize, (IpyEscapeKind, Box<str>), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -33653,10 +33653,10 @@ fn __action74<
} }
)) ))
} else { } else {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location, location,
})? ))?
} }
} }
} }
@ -33668,7 +33668,7 @@ fn __action75<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, c, _): (TextSize, (IpyEscapeKind, String), TextSize), (_, c, _): (TextSize, (IpyEscapeKind, Box<str>), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -33676,10 +33676,10 @@ fn __action75<
if mode == Mode::Ipython { if mode == Mode::Ipython {
// This should never occur as the lexer won't allow it. // This should never occur as the lexer won't allow it.
if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) { if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()), LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()),
location, location,
})?; ))?;
} }
Ok(ast::ExprIpyEscapeCommand { Ok(ast::ExprIpyEscapeCommand {
kind: c.0, kind: c.0,
@ -33687,10 +33687,10 @@ fn __action75<
range: (location..end_location).into() range: (location..end_location).into()
}.into()) }.into())
} else { } else {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location, location,
})? ))?
} }
} }
} }
@ -33715,10 +33715,10 @@ fn __action76<
}, },
ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => { ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => {
let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else { let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()), LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()),
location: range.start(), range.start(),
}); ));
}; };
unparse_expr(value, buffer)?; unparse_expr(value, buffer)?;
buffer.push('['); buffer.push('[');
@ -33731,10 +33731,10 @@ fn __action76<
buffer.push_str(attr.as_str()); buffer.push_str(attr.as_str());
}, },
_ => { _ => {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()), LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()),
location: expr.start(), expr.start(),
}); ));
} }
} }
Ok(()) Ok(())
@ -33742,10 +33742,10 @@ fn __action76<
if mode != Mode::Ipython { if mode != Mode::Ipython {
return Err(ParseError::User { return Err(ParseError::User {
error: LexicalError { error: LexicalError::new(
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location, location,
}, ),
}); });
} }
@ -33754,10 +33754,10 @@ fn __action76<
2 => IpyEscapeKind::Help2, 2 => IpyEscapeKind::Help2,
_ => { _ => {
return Err(ParseError::User { return Err(ParseError::User {
error: LexicalError { error: LexicalError::new(
error: LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()), LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()),
location, location,
}, ),
}); });
} }
}; };
@ -33768,7 +33768,7 @@ fn __action76<
Ok(ast::Stmt::IpyEscapeCommand( Ok(ast::Stmt::IpyEscapeCommand(
ast::StmtIpyEscapeCommand { ast::StmtIpyEscapeCommand {
kind, kind,
value, value: value.into_boxed_str(),
range: (location..end_location).into() range: (location..end_location).into()
} }
)) ))
@ -34126,10 +34126,10 @@ fn __action95<
{ {
{ {
if name.as_str() == "_" { if name.as_str() == "_" {
Err(LexicalError { Err(LexicalError::new(
error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()), LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()),
location, location,
})? ))?
} else { } else {
Ok(ast::Pattern::MatchAs( Ok(ast::Pattern::MatchAs(
ast::PatternMatchAs { ast::PatternMatchAs {
@ -35910,17 +35910,17 @@ fn __action184<
(_, parameters, _): (TextSize, core::option::Option<ast::Parameters>, TextSize), (_, parameters, _): (TextSize, core::option::Option<ast::Parameters>, TextSize),
(_, end_location_args, _): (TextSize, TextSize, TextSize), (_, end_location_args, _): (TextSize, TextSize, TextSize),
(_, _, _): (TextSize, token::Tok, TextSize), (_, _, _): (TextSize, token::Tok, TextSize),
(_, fstring_middle, _): (TextSize, core::option::Option<(String, bool, bool)>, TextSize), (_, fstring_middle, _): (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
(_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize), (_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
{ {
if fstring_middle.is_some() { if fstring_middle.is_some() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location, location,
})?; ))?;
} }
parameters.as_ref().map(validate_arguments).transpose()?; parameters.as_ref().map(validate_arguments).transpose()?;
@ -36363,7 +36363,7 @@ fn __action217<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, string, _): (TextSize, (String, StringKind, bool), TextSize), (_, string, _): (TextSize, (Box<str>, StringKind, bool), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -36413,7 +36413,7 @@ fn __action220<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, fstring_middle, _): (TextSize, (String, bool, bool), TextSize), (_, fstring_middle, _): (TextSize, (Box<str>, bool, bool), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -36441,10 +36441,10 @@ fn __action221<
{ {
{ {
if value.expr.is_lambda_expr() && !value.is_parenthesized() { if value.expr.is_lambda_expr() && !value.is_parenthesized() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses), LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location: value.start(), value.start(),
})?; ))?;
} }
let debug_text = debug.map(|_| { let debug_text = debug.map(|_| {
let start_offset = location + "{".text_len(); let start_offset = location + "{".text_len();
@ -36514,18 +36514,18 @@ fn __action224<
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, _, _): (TextSize, token::Tok, TextSize), (_, _, _): (TextSize, token::Tok, TextSize),
(_, name_location, _): (TextSize, TextSize, TextSize), (_, name_location, _): (TextSize, TextSize, TextSize),
(_, s, _): (TextSize, String, TextSize), (_, s, _): (TextSize, Box<str>, TextSize),
) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
{ {
let conversion = match s.as_str() { let conversion = match s.as_ref() {
"s" => ast::ConversionFlag::Str, "s" => ast::ConversionFlag::Str,
"r" => ast::ConversionFlag::Repr, "r" => ast::ConversionFlag::Repr,
"a" => ast::ConversionFlag::Ascii, "a" => ast::ConversionFlag::Ascii,
_ => Err(LexicalError { _ => Err(LexicalError::new(
error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag), LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
location: name_location, name_location,
})? ))?
}; };
Ok((location, conversion)) Ok((location, conversion))
} }
@ -36899,7 +36899,7 @@ fn __action249<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize), (_, location, _): (TextSize, TextSize, TextSize),
(_, s, _): (TextSize, String, TextSize), (_, s, _): (TextSize, Box<str>, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
@ -37357,8 +37357,8 @@ fn __action281<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
(_, __0, _): (TextSize, (String, bool, bool), TextSize), (_, __0, _): (TextSize, (Box<str>, bool, bool), TextSize),
) -> core::option::Option<(String, bool, bool)> ) -> core::option::Option<(Box<str>, bool, bool)>
{ {
Some(__0) Some(__0)
} }
@ -37371,7 +37371,7 @@ fn __action282<
mode: Mode, mode: Mode,
__lookbehind: &TextSize, __lookbehind: &TextSize,
__lookahead: &TextSize, __lookahead: &TextSize,
) -> core::option::Option<(String, bool, bool)> ) -> core::option::Option<(Box<str>, bool, bool)>
{ {
None None
} }
@ -39668,10 +39668,10 @@ fn __action445<
{ {
{ {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
location, location,
})?; ))?;
} }
let kwarg = kwarg.flatten(); let kwarg = kwarg.flatten();
@ -39793,10 +39793,10 @@ fn __action453<
{ {
{ {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
location, location,
})?; ))?;
} }
let kwarg = kwarg.flatten(); let kwarg = kwarg.flatten();
@ -41296,10 +41296,10 @@ fn __action554<
{ {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.expr.is_starred_expr() { if mid.expr.is_starred_expr() {
return Err(LexicalError{ return Err(LexicalError::new(
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
location: mid.start(), mid.start(),
})?; ))?;
} }
Ok(crate::parser::ParenthesizedExpr { Ok(crate::parser::ParenthesizedExpr {
expr: mid.into(), expr: mid.into(),
@ -41386,10 +41386,10 @@ fn __action558<
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
{ {
Err(LexicalError{ Err(LexicalError::new(
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
location, location,
}.into()) ).into())
} }
} }
@ -41994,10 +41994,10 @@ fn __action596<
{ {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.expr.is_starred_expr() { if mid.expr.is_starred_expr() {
return Err(LexicalError{ return Err(LexicalError::new(
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
location: mid.start(), mid.start(),
})?; ))?;
} }
Ok(crate::parser::ParenthesizedExpr { Ok(crate::parser::ParenthesizedExpr {
expr: mid.into(), expr: mid.into(),
@ -42084,10 +42084,10 @@ fn __action600<
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
{ {
Err(LexicalError{ Err(LexicalError::new(
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
location, location,
}.into()) ).into())
} }
} }
@ -48027,7 +48027,7 @@ fn __action789<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, String, TextSize), __0: (TextSize, Box<str>, TextSize),
__1: (TextSize, TextSize, TextSize), __1: (TextSize, TextSize, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
@ -48055,7 +48055,7 @@ fn __action790<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, String, TextSize), __0: (TextSize, Box<str>, TextSize),
__1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), __1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize),
__2: (TextSize, TextSize, TextSize), __2: (TextSize, TextSize, TextSize),
) -> ast::Identifier ) -> ast::Identifier
@ -48408,7 +48408,7 @@ fn __action801<
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, token::Tok, TextSize), __0: (TextSize, token::Tok, TextSize),
__1: (TextSize, String, TextSize), __1: (TextSize, Box<str>, TextSize),
) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
let __start0 = __0.0; let __start0 = __0.0;
@ -48505,7 +48505,7 @@ fn __action804<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (String, bool, bool), TextSize), __0: (TextSize, (Box<str>, bool, bool), TextSize),
__1: (TextSize, TextSize, TextSize), __1: (TextSize, TextSize, TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -49209,7 +49209,7 @@ fn __action826<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, String, TextSize), __0: (TextSize, Box<str>, TextSize),
__1: (TextSize, TextSize, TextSize), __1: (TextSize, TextSize, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
@ -49519,7 +49519,7 @@ fn __action836<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize), __0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
__1: (TextSize, TextSize, TextSize), __1: (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -49547,7 +49547,7 @@ fn __action837<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize), __0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
__1: (TextSize, TextSize, TextSize), __1: (TextSize, TextSize, TextSize),
) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -49609,7 +49609,7 @@ fn __action839<
__1: (TextSize, core::option::Option<ast::Parameters>, TextSize), __1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
__2: (TextSize, TextSize, TextSize), __2: (TextSize, TextSize, TextSize),
__3: (TextSize, token::Tok, TextSize), __3: (TextSize, token::Tok, TextSize),
__4: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), __4: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__5: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __5: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
__6: (TextSize, TextSize, TextSize), __6: (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
@ -52719,7 +52719,7 @@ fn __action937<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (String, StringKind, bool), TextSize), __0: (TextSize, (Box<str>, StringKind, bool), TextSize),
__1: (TextSize, TextSize, TextSize), __1: (TextSize, TextSize, TextSize),
) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -64211,7 +64211,7 @@ fn __action1304<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, String, TextSize), __0: (TextSize, Box<str>, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
let __start0 = __0.2; let __start0 = __0.2;
@ -64237,7 +64237,7 @@ fn __action1305<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, String, TextSize), __0: (TextSize, Box<str>, TextSize),
__1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), __1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
@ -64527,7 +64527,7 @@ fn __action1315<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (String, bool, bool), TextSize), __0: (TextSize, (Box<str>, bool, bool), TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
let __start0 = __0.2; let __start0 = __0.2;
@ -65035,7 +65035,7 @@ fn __action1333<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, String, TextSize), __0: (TextSize, Box<str>, TextSize),
) -> ast::Identifier ) -> ast::Identifier
{ {
let __start0 = __0.2; let __start0 = __0.2;
@ -65347,7 +65347,7 @@ fn __action1344<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize), __0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
let __start0 = __0.2; let __start0 = __0.2;
@ -65373,7 +65373,7 @@ fn __action1345<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize), __0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
let __start0 = __0.2; let __start0 = __0.2;
@ -65430,7 +65430,7 @@ fn __action1347<
__0: (TextSize, token::Tok, TextSize), __0: (TextSize, token::Tok, TextSize),
__1: (TextSize, core::option::Option<ast::Parameters>, TextSize), __1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
__2: (TextSize, token::Tok, TextSize), __2: (TextSize, token::Tok, TextSize),
__3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), __3: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -69997,7 +69997,7 @@ fn __action1494<
>( >(
source_code: &str, source_code: &str,
mode: Mode, mode: Mode,
__0: (TextSize, (String, StringKind, bool), TextSize), __0: (TextSize, (Box<str>, StringKind, bool), TextSize),
) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
let __start0 = __0.2; let __start0 = __0.2;
@ -77662,7 +77662,7 @@ fn __action1727<
__0: (TextSize, token::Tok, TextSize), __0: (TextSize, token::Tok, TextSize),
__1: (TextSize, ast::Parameters, TextSize), __1: (TextSize, ast::Parameters, TextSize),
__2: (TextSize, token::Tok, TextSize), __2: (TextSize, token::Tok, TextSize),
__3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), __3: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -77693,7 +77693,7 @@ fn __action1728<
mode: Mode, mode: Mode,
__0: (TextSize, token::Tok, TextSize), __0: (TextSize, token::Tok, TextSize),
__1: (TextSize, token::Tok, TextSize), __1: (TextSize, token::Tok, TextSize),
__2: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), __2: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__3: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -79598,7 +79598,7 @@ fn __action1785<
__0: (TextSize, token::Tok, TextSize), __0: (TextSize, token::Tok, TextSize),
__1: (TextSize, ast::Parameters, TextSize), __1: (TextSize, ast::Parameters, TextSize),
__2: (TextSize, token::Tok, TextSize), __2: (TextSize, token::Tok, TextSize),
__3: (TextSize, (String, bool, bool), TextSize), __3: (TextSize, (Box<str>, bool, bool), TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {
@ -79661,7 +79661,7 @@ fn __action1787<
mode: Mode, mode: Mode,
__0: (TextSize, token::Tok, TextSize), __0: (TextSize, token::Tok, TextSize),
__1: (TextSize, token::Tok, TextSize), __1: (TextSize, token::Tok, TextSize),
__2: (TextSize, (String, bool, bool), TextSize), __2: (TextSize, (Box<str>, bool, bool), TextSize),
__3: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>> ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{ {

2
crates/ruff_python_parser/src/soft_keywords.rs
View file

@ -203,7 +203,7 @@ fn soft_to_name(tok: &Tok) -> Tok {
_ => unreachable!("other tokens never reach here"), _ => unreachable!("other tokens never reach here"),
}; };
Tok::Name { Tok::Name {
name: name.to_owned(), name: name.to_string().into_boxed_str(),
} }
} }

50
crates/ruff_python_parser/src/string.rs
View file

@ -151,10 +151,10 @@ impl<'a> StringParser<'a> {
fn parse_escaped_char(&mut self, string: &mut String) -> Result<(), LexicalError> { fn parse_escaped_char(&mut self, string: &mut String) -> Result<(), LexicalError> {
let Some(first_char) = self.next_char() else { let Some(first_char) = self.next_char() else {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::StringError, LexicalErrorType::StringError,
location: self.get_pos(), self.get_pos(),
}); ));
}; };
let new_char = match first_char { let new_char = match first_char {
@ -184,12 +184,14 @@ impl<'a> StringParser<'a> {
} }
_ => { _ => {
if self.kind.is_any_bytes() && !first_char.is_ascii() { if self.kind.is_any_bytes() && !first_char.is_ascii() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError( LexicalErrorType::OtherError(
"bytes can only contain ASCII literal characters".to_owned(), "bytes can only contain ASCII literal characters"
.to_string()
.into_boxed_str(),
), ),
location: self.get_pos(), self.get_pos(),
}); ));
} }
string.push('\\'); string.push('\\');
@ -257,7 +259,9 @@ impl<'a> StringParser<'a> {
if !ch.is_ascii() { if !ch.is_ascii() {
return Err(LexicalError::new( return Err(LexicalError::new(
LexicalErrorType::OtherError( LexicalErrorType::OtherError(
"bytes can only contain ASCII literal characters".to_string(), "bytes can only contain ASCII literal characters"
.to_string()
.into_boxed_str(),
), ),
self.get_pos(), self.get_pos(),
)); ));
@ -291,7 +295,7 @@ impl<'a> StringParser<'a> {
} }
} }
Ok(StringType::Str(ast::StringLiteral { Ok(StringType::Str(ast::StringLiteral {
value, value: value.into_boxed_str(),
unicode: self.kind.is_unicode(), unicode: self.kind.is_unicode(),
range: self.range, range: self.range,
})) }))
@ -354,12 +358,14 @@ pub(crate) fn concatenated_strings(
let has_bytes = byte_literal_count > 0; let has_bytes = byte_literal_count > 0;
if has_bytes && byte_literal_count < strings.len() { if has_bytes && byte_literal_count < strings.len() {
return Err(LexicalError { return Err(LexicalError::new(
error: LexicalErrorType::OtherError( LexicalErrorType::OtherError(
"cannot mix bytes and nonbytes literals".to_owned(), "cannot mix bytes and nonbytes literals"
.to_string()
.into_boxed_str(),
), ),
location: range.start(), range.start(),
}); ));
} }
if has_bytes { if has_bytes {
@ -418,15 +424,12 @@ struct FStringError {
impl From<FStringError> for LexicalError { impl From<FStringError> for LexicalError {
fn from(err: FStringError) -> Self { fn from(err: FStringError) -> Self {
LexicalError { LexicalError::new(LexicalErrorType::FStringError(err.error), err.location)
error: LexicalErrorType::FStringError(err.error),
location: err.location,
}
} }
} }
/// Represents the different types of errors that can occur during parsing of an f-string. /// Represents the different types of errors that can occur during parsing of an f-string.
#[derive(Debug, Clone, PartialEq)] #[derive(Copy, Debug, Clone, PartialEq)]
pub enum FStringErrorType { pub enum FStringErrorType {
/// Expected a right brace after an opened left brace. /// Expected a right brace after an opened left brace.
UnclosedLbrace, UnclosedLbrace,
@ -466,10 +469,7 @@ impl std::fmt::Display for FStringErrorType {
impl From<FStringError> for crate::parser::LalrpopError<TextSize, Tok, LexicalError> { impl From<FStringError> for crate::parser::LalrpopError<TextSize, Tok, LexicalError> {
fn from(err: FStringError) -> Self { fn from(err: FStringError) -> Self {
lalrpop_util::ParseError::User { lalrpop_util::ParseError::User {
error: LexicalError { error: LexicalError::new(LexicalErrorType::FStringError(err.error), err.location),
error: LexicalErrorType::FStringError(err.error),
location: err.location,
},
} }
} }
} }

21
crates/ruff_python_parser/src/token.rs
View file

@ -16,7 +16,7 @@ pub enum Tok {
/// Token value for a name, commonly known as an identifier. /// Token value for a name, commonly known as an identifier.
Name { Name {
/// The name value. /// The name value.
name: String, name: Box<str>,
}, },
/// Token value for an integer. /// Token value for an integer.
Int { Int {
@ -38,7 +38,7 @@ pub enum Tok {
/// Token value for a string. /// Token value for a string.
String { String {
/// The string value. /// The string value.
value: String, value: Box<str>,
/// The kind of string. /// The kind of string.
kind: StringKind, kind: StringKind,
/// Whether the string is triple quoted. /// Whether the string is triple quoted.
@ -51,7 +51,7 @@ pub enum Tok {
/// part of the expression part and isn't an opening or closing brace. /// part of the expression part and isn't an opening or closing brace.
FStringMiddle { FStringMiddle {
/// The string value. /// The string value.
value: String, value: Box<str>,
/// Whether the string is raw or not. /// Whether the string is raw or not.
is_raw: bool, is_raw: bool,
/// Whether the string is triple quoted. /// Whether the string is triple quoted.
@ -63,12 +63,12 @@ pub enum Tok {
/// only when the mode is [`Mode::Ipython`]. /// only when the mode is [`Mode::Ipython`].
IpyEscapeCommand { IpyEscapeCommand {
/// The magic command value. /// The magic command value.
value: String, value: Box<str>,
/// The kind of magic command. /// The kind of magic command.
kind: IpyEscapeKind, kind: IpyEscapeKind,
}, },
/// Token value for a comment. These are filtered out of the token stream prior to parsing. /// Token value for a comment. These are filtered out of the token stream prior to parsing.
Comment(String), Comment(Box<str>),
/// Token value for a newline. /// Token value for a newline.
Newline, Newline,
/// Token value for a newline that is not a logical line break. These are filtered out of /// Token value for a newline that is not a logical line break. These are filtered out of
@ -912,3 +912,14 @@ impl From<&Tok> for TokenKind {
Self::from_token(value) Self::from_token(value)
} }
} }
#[cfg(target_pointer_width = "64")]
mod sizes {
use crate::lexer::{LexicalError, LexicalErrorType};
use crate::Tok;
use static_assertions::assert_eq_size;
assert_eq_size!(Tok, [u8; 24]);
assert_eq_size!(LexicalErrorType, [u8; 24]);
assert_eq_size!(Result<Tok, LexicalError>, [u8; 32]);
}

2
fuzz/fuzz_targets/ruff_parse_simple.rs
View file

@ -47,7 +47,7 @@ fn do_fuzz(case: &[u8]) -> Corpus {
); );
} }
Err(err) => { Err(err) => {
let offset = err.location.to_usize(); let offset = err.location().to_usize();
assert!( assert!(
code.is_char_boundary(offset), code.is_char_boundary(offset),
"Invalid error location {} (not at char boundary)", "Invalid error location {} (not at char boundary)",