mirrors/ruff
1
0
Fork 0
mirror of https://github.com/astral-sh/ruff.git synced 2025-07-12 07:35:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Reduce Result<Tok, LexicalError> size by using Box<str> instead of String (#9885)

Browse source
This commit is contained in:
Micha Reiser 2024-02-08 21:36:22 +01:00 • committed by GitHub
parent 9027169125
commit fe7d965334
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 454 additions and 425 deletions
Download patch file Download diff file Expand all files Collapse all files

64
crates/ruff_python_parser/src/function.rs
View file

@ -39,10 +39,10 @@ pub(crate) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), Lexi
let range = arg.range;
let arg_name = arg.name.as_str();
if !all_arg_names.insert(arg_name) {
return Err(LexicalError {
error: LexicalErrorType::DuplicateArgumentError(arg_name.to_string()),
location: range.start(),
});
return Err(LexicalError::new(
LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()),
range.start(),
));
}
}
@ -64,10 +64,10 @@ pub(crate) fn validate_pos_params(
.skip_while(|arg| arg.default.is_some()) // and then args with default
.next(); // there must not be any more args without default
if let Some(invalid) = first_invalid {
return Err(LexicalError {
error: LexicalErrorType::DefaultArgumentError,
location: invalid.parameter.start(),
});
return Err(LexicalError::new(
LexicalErrorType::DefaultArgumentError,
invalid.parameter.start(),
));
}
Ok(())
}
@ -94,12 +94,12 @@ pub(crate) fn parse_arguments(
// Check for duplicate keyword arguments in the call.
if let Some(keyword_name) = &name {
if !keyword_names.insert(keyword_name.to_string()) {
return Err(LexicalError {
error: LexicalErrorType::DuplicateKeywordArgumentError(
keyword_name.to_string(),
return Err(LexicalError::new(
LexicalErrorType::DuplicateKeywordArgumentError(
keyword_name.to_string().into_boxed_str(),
),
location: start,
});
start,
));
}
} else {
double_starred = true;
@ -113,17 +113,17 @@ pub(crate) fn parse_arguments(
} else {
// Positional arguments mustn't follow keyword arguments.
if !keywords.is_empty() && !is_starred(&value) {
return Err(LexicalError {
error: LexicalErrorType::PositionalArgumentError,
location: value.start(),
});
return Err(LexicalError::new(
LexicalErrorType::PositionalArgumentError,
value.start(),
));
// Allow starred arguments after keyword arguments but
// not after double-starred arguments.
} else if double_starred {
return Err(LexicalError {
error: LexicalErrorType::UnpackedArgumentError,
location: value.start(),
});
return Err(LexicalError::new(
LexicalErrorType::UnpackedArgumentError,
value.start(),
));
}
args.push(value);
@ -202,22 +202,22 @@ mod tests {
function_and_lambda_error! {
// Check definitions
test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()),
test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()),
test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError,
test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError,
// Check some calls.
test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError,
test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError,
test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()),
test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string().into_boxed_str()),
}
}

2
crates/ruff_python_parser/src/invalid.rs
View file

@ -39,7 +39,7 @@ pub(crate) fn assignment_target(target: &Expr) -> Result<(), LexicalError> {
let err = |location: TextSize| -> LexicalError {
let error = LexicalErrorType::AssignmentError;
LexicalError { error, location }
LexicalError::new(error, location)
};
match *target {
BoolOp(ref e) => Err(err(e.range.start())),

244
crates/ruff_python_parser/src/lexer.rs
View file

@ -107,10 +107,10 @@ where
fn next(&mut self) -> Option<Self::Item> {
let result = match self.inner.next()? {
Ok((tok, range)) => Ok((tok, range + self.start_offset)),
Err(error) => Err(LexicalError {
location: error.location + self.start_offset,
..error
}),
Err(error) => {
let location = error.location() + self.start_offset;
Err(LexicalError::new(error.into_error(), location))
}
};
Some(result)
@ -241,7 +241,7 @@ impl<'source> Lexer<'source> {
"yield" => Tok::Yield,
_ => {
return Ok(Tok::Name {
name: text.to_string(),
name: text.to_string().into_boxed_str(),
})
}
};
@ -284,10 +284,10 @@ impl<'source> Lexer<'source> {
let value = match Int::from_str_radix(number.as_str(), radix.as_u32(), token) {
Ok(int) => int,
Err(err) => {
return Err(LexicalError {
error: LexicalErrorType::OtherError(format!("{err:?}")),
location: self.token_range().start(),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()),
self.token_range().start(),
));
}
};
Ok(Tok::Int { value })
@ -309,10 +309,10 @@ impl<'source> Lexer<'source> {
number.push('.');
if self.cursor.eat_char('_') {
return Err(LexicalError {
error: LexicalErrorType::OtherError("Invalid Syntax".to_owned()),
location: self.offset() - TextSize::new(1),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError("Invalid Syntax".to_string().into_boxed_str()),
self.offset() - TextSize::new(1),
));
}
self.radix_run(&mut number, Radix::Decimal);
@ -340,9 +340,13 @@ impl<'source> Lexer<'source> {
if is_float {
// Improvement: Use `Cow` instead of pushing to value text
let value = f64::from_str(number.as_str()).map_err(|_| LexicalError {
error: LexicalErrorType::OtherError("Invalid decimal literal".to_owned()),
location: self.token_start(),
let value = f64::from_str(number.as_str()).map_err(|_| {
LexicalError::new(
LexicalErrorType::OtherError(
"Invalid decimal literal".to_string().into_boxed_str(),
),
self.token_start(),
)
})?;
// Parse trailing 'j':
@ -364,18 +368,20 @@ impl<'source> Lexer<'source> {
Ok(value) => {
if start_is_zero && value.as_u8() != Some(0) {
// Leading zeros in decimal integer literals are not permitted.
return Err(LexicalError {
error: LexicalErrorType::OtherError("Invalid Token".to_owned()),
location: self.token_range().start(),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError(
"Invalid Token".to_string().into_boxed_str(),
),
self.token_range().start(),
));
}
value
}
Err(err) => {
return Err(LexicalError {
error: LexicalErrorType::OtherError(format!("{err:?}")),
location: self.token_range().start(),
})
return Err(LexicalError::new(
LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()),
self.token_range().start(),
))
}
};
Ok(Tok::Int { value })
@ -411,7 +417,7 @@ impl<'source> Lexer<'source> {
let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len());
self.cursor.skip_bytes(offset);
Tok::Comment(self.token_text().to_string())
Tok::Comment(self.token_text().to_string().into_boxed_str())
}
/// Lex a single IPython escape command.
@ -508,12 +514,15 @@ impl<'source> Lexer<'source> {
2 => IpyEscapeKind::Help2,
_ => unreachable!("`question_count` is always 1 or 2"),
};
return Tok::IpyEscapeCommand { kind, value };
return Tok::IpyEscapeCommand {
kind,
value: value.into_boxed_str(),
};
}
'\n' | '\r' | EOF_CHAR => {
return Tok::IpyEscapeCommand {
kind: escape_kind,
value,
value: value.into_boxed_str(),
};
}
c => {
@ -584,10 +593,10 @@ impl<'source> Lexer<'source> {
} else {
FStringErrorType::UnterminatedString
};
return Err(LexicalError {
error: LexicalErrorType::FStringError(error),
location: self.offset(),
});
return Err(LexicalError::new(
LexicalErrorType::FStringError(error),
self.offset(),
));
}
'\n' | '\r' if !fstring.is_triple_quoted() => {
// If we encounter a newline while we're in a format spec, then
@ -597,10 +606,10 @@ impl<'source> Lexer<'source> {
if in_format_spec {
break;
}
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::UnterminatedString),
location: self.offset(),
});
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::UnterminatedString),
self.offset(),
));
}
'\\' => {
self.cursor.bump(); // '\'
@ -673,7 +682,7 @@ impl<'source> Lexer<'source> {
normalized
};
Ok(Some(Tok::FStringMiddle {
value,
value: value.into_boxed_str(),
is_raw: fstring.is_raw_string(),
triple_quoted: fstring.is_triple_quoted(),
}))
@ -705,18 +714,16 @@ impl<'source> Lexer<'source> {
if fstring.quote_char() == quote
&& fstring.is_triple_quoted() == triple_quoted
{
return Err(LexicalError {
error: LexicalErrorType::FStringError(
FStringErrorType::UnclosedLbrace,
),
location: self.cursor.text_len(),
});
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace),
self.cursor.text_len(),
));
}
}
return Err(LexicalError {
error: LexicalErrorType::Eof,
location: self.cursor.text_len(),
});
return Err(LexicalError::new(
LexicalErrorType::Eof,
self.cursor.text_len(),
));
};
// Rare case: if there are an odd number of backslashes before the quote, then
@ -756,18 +763,16 @@ impl<'source> Lexer<'source> {
if fstring.quote_char() == quote
&& fstring.is_triple_quoted() == triple_quoted
{
return Err(LexicalError {
error: LexicalErrorType::FStringError(
FStringErrorType::UnclosedLbrace,
),
location: self.offset(),
});
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::UnclosedLbrace),
self.offset(),
));
}
}
return Err(LexicalError {
error: LexicalErrorType::StringError,
location: self.offset(),
});
return Err(LexicalError::new(
LexicalErrorType::StringError,
self.offset(),
));
};
// Rare case: if there are an odd number of backslashes before the quote, then
@ -797,20 +802,22 @@ impl<'source> Lexer<'source> {
// matches with f-strings quotes and if it is, then this must be a
// missing '}' token so raise the proper error.
if fstring.quote_char() == quote && !fstring.is_triple_quoted() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(
return Err(LexicalError::new(
LexicalErrorType::FStringError(
FStringErrorType::UnclosedLbrace,
),
location: self.offset() - TextSize::new(1),
});
self.offset() - TextSize::new(1),
));
}
}
return Err(LexicalError {
error: LexicalErrorType::OtherError(
"EOL while scanning string literal".to_owned(),
return Err(LexicalError::new(
LexicalErrorType::OtherError(
"EOL while scanning string literal"
.to_string()
.into_boxed_str(),
),
location: self.offset() - TextSize::new(1),
});
self.offset() - TextSize::new(1),
));
}
Some(ch) if ch == quote => {
break self.offset() - TextSize::new(1);
@ -821,7 +828,9 @@ impl<'source> Lexer<'source> {
};
Ok(Tok::String {
value: self.source[TextRange::new(value_start, value_end)].to_string(),
value: self.source[TextRange::new(value_start, value_end)]
.to_string()
.into_boxed_str(),
kind,
triple_quoted,
})
@ -889,10 +898,10 @@ impl<'source> Lexer<'source> {
Ok((identifier, self.token_range()))
} else {
Err(LexicalError {
error: LexicalErrorType::UnrecognizedToken { tok: c },
location: self.token_start(),
})
Err(LexicalError::new(
LexicalErrorType::UnrecognizedToken { tok: c },
self.token_start(),
))
}
} else {
// Reached the end of the file. Emit a trailing newline token if not at the beginning of a logical line,
@ -915,15 +924,12 @@ impl<'source> Lexer<'source> {
if self.cursor.eat_char('\r') {
self.cursor.eat_char('\n');
} else if self.cursor.is_eof() {
return Err(LexicalError {
error: LexicalErrorType::Eof,
location: self.token_start(),
});
return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start()));
} else if !self.cursor.eat_char('\n') {
return Err(LexicalError {
error: LexicalErrorType::LineContinuationError,
location: self.token_start(),
});
return Err(LexicalError::new(
LexicalErrorType::LineContinuationError,
self.token_start(),
));
}
}
// Form feed
@ -956,15 +962,12 @@ impl<'source> Lexer<'source> {
if self.cursor.eat_char('\r') {
self.cursor.eat_char('\n');
} else if self.cursor.is_eof() {
return Err(LexicalError {
error: LexicalErrorType::Eof,
location: self.token_start(),
});
return Err(LexicalError::new(LexicalErrorType::Eof, self.token_start()));
} else if !self.cursor.eat_char('\n') {
return Err(LexicalError {
error: LexicalErrorType::LineContinuationError,
location: self.token_start(),
});
return Err(LexicalError::new(
LexicalErrorType::LineContinuationError,
self.token_start(),
));
}
indentation = Indentation::root();
}
@ -1015,10 +1018,10 @@ impl<'source> Lexer<'source> {
Some((Tok::Indent, self.token_range()))
}
Err(_) => {
return Err(LexicalError {
error: LexicalErrorType::IndentationError,
location: self.offset(),
});
return Err(LexicalError::new(
LexicalErrorType::IndentationError,
self.offset(),
));
}
};
@ -1031,10 +1034,7 @@ impl<'source> Lexer<'source> {
if self.nesting > 0 {
// Reset the nesting to avoid going into infinite loop.
self.nesting = 0;
return Err(LexicalError {
error: LexicalErrorType::Eof,
location: self.offset(),
});
return Err(LexicalError::new(LexicalErrorType::Eof, self.offset()));
}
// Next, insert a trailing newline, if required.
@ -1199,10 +1199,10 @@ impl<'source> Lexer<'source> {
'}' => {
if let Some(fstring) = self.fstrings.current_mut() {
if fstring.nesting() == self.nesting {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::SingleRbrace),
location: self.token_start(),
});
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::SingleRbrace),
self.token_start(),
));
}
fstring.try_end_format_spec(self.nesting);
}
@ -1293,10 +1293,10 @@ impl<'source> Lexer<'source> {
_ => {
self.state = State::Other;
return Err(LexicalError {
error: LexicalErrorType::UnrecognizedToken { tok: c },
location: self.token_start(),
});
return Err(LexicalError::new(
LexicalErrorType::UnrecognizedToken { tok: c },
self.token_start(),
));
}
};
@ -1357,9 +1357,9 @@ impl FusedIterator for Lexer<'_> {}
#[derive(Debug, Clone, PartialEq)]
pub struct LexicalError {
/// The type of error that occurred.
pub error: LexicalErrorType,
error: LexicalErrorType,
/// The location of the error.
pub location: TextSize,
location: TextSize,
}
impl LexicalError {
@ -1367,19 +1367,31 @@ impl LexicalError {
pub fn new(error: LexicalErrorType, location: TextSize) -> Self {
Self { error, location }
}
pub fn error(&self) -> &LexicalErrorType {
&self.error
}
pub fn into_error(self) -> LexicalErrorType {
self.error
}
pub fn location(&self) -> TextSize {
self.location
}
}
impl std::ops::Deref for LexicalError {
type Target = LexicalErrorType;
fn deref(&self) -> &Self::Target {
&self.error
self.error()
}
}
impl std::error::Error for LexicalError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
Some(self.error())
}
}
@ -1388,8 +1400,8 @@ impl std::fmt::Display for LexicalError {
write!(
f,
"{} at byte offset {}",
&self.error,
u32::from(self.location)
self.error(),
u32::from(self.location())
)
}
}
@ -1397,6 +1409,9 @@ impl std::fmt::Display for LexicalError {
/// Represents the different types of errors that can occur during lexing.
#[derive(Debug, Clone, PartialEq)]
pub enum LexicalErrorType {
/// A duplicate argument was found in a function definition.
DuplicateArgumentError(Box<str>),
// TODO: Can probably be removed, the places it is used seem to be able
// to use the `UnicodeError` variant instead.
#[doc(hidden)]
@ -1414,14 +1429,13 @@ pub enum LexicalErrorType {
TabsAfterSpaces,
/// A non-default argument follows a default argument.
DefaultArgumentError,
/// A duplicate argument was found in a function definition.
DuplicateArgumentError(String),
/// A positional argument follows a keyword argument.
PositionalArgumentError,
/// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`.
UnpackedArgumentError,
/// A keyword argument was repeated.
DuplicateKeywordArgumentError(String),
DuplicateKeywordArgumentError(Box<str>),
/// An unrecognized token was encountered.
UnrecognizedToken { tok: char },
/// An f-string error containing the [`FStringErrorType`].
@ -1433,7 +1447,7 @@ pub enum LexicalErrorType {
/// Occurs when a syntactically invalid assignment was encountered.
AssignmentError,
/// An unexpected error occurred.
OtherError(String),
OtherError(Box<str>),
}
impl std::error::Error for LexicalErrorType {}
@ -2053,8 +2067,8 @@ def f(arg=%timeit a = b):
match lexed.as_slice() {
[Err(error)] => {
assert_eq!(
error.error,
LexicalErrorType::UnrecognizedToken { tok: '🐦' }
error.error(),
&LexicalErrorType::UnrecognizedToken { tok: '🐦' }
);
}
result => panic!("Expected an error token but found {result:?}"),
@ -2267,7 +2281,7 @@ f"{(lambda x:{x})}"
}
fn lex_fstring_error(source: &str) -> FStringErrorType {
match lex_error(source).error {
match lex_error(source).into_error() {
LexicalErrorType::FStringError(error) => error,
err => panic!("Expected FStringError: {err:?}"),
}

8
crates/ruff_python_parser/src/parser.rs
View file

@ -285,8 +285,8 @@ fn parse_error_from_lalrpop(err: LalrpopError<TextSize, Tok, LexicalError>) -> P
offset: token.0,
},
LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error),
offset: error.location,
offset: error.location(),
error: ParseErrorType::Lexical(error.into_error()),
},
LalrpopError::UnrecognizedToken { token, expected } => {
// Hacky, but it's how CPython does it. See PyParser_AddToken,
@ -359,8 +359,8 @@ impl ParseErrorType {
impl From<LexicalError> for ParseError {
fn from(error: LexicalError) -> Self {
ParseError {
error: ParseErrorType::Lexical(error.error),
offset: error.location,
offset: error.location(),
error: ParseErrorType::Lexical(error.into_error()),
}
}
}

108
crates/ruff_python_parser/src/python.lalrpop
View file

@ -289,7 +289,7 @@ ImportAsAlias<I>: ast::Alias = {
DottedName: ast::Identifier = {
<location:@L> <n:name> <end_location:@R> => ast::Identifier::new(n, (location..end_location).into()),
<location:@L> <n:name> <n2: ("." Identifier)+> <end_location:@R> => {
let mut r = n;
let mut r = String::from(n);
for x in n2 {
r.push('.');
r.push_str(x.1.as_str());
@ -337,10 +337,10 @@ IpyEscapeCommandStatement: ast::Stmt = {
}
))
} else {
Err(LexicalError {
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location,
})?
))?
}
}
}
@ -350,10 +350,10 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = {
if mode == Mode::Ipython {
// This should never occur as the lexer won't allow it.
if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) {
return Err(LexicalError {
error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()),
return Err(LexicalError::new(
LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()),
location,
})?;
))?;
}
Ok(ast::ExprIpyEscapeCommand {
kind: c.0,
@ -361,10 +361,10 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = {
range: (location..end_location).into()
}.into())
} else {
Err(LexicalError {
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location,
})?
))?
}
}
}
@ -381,10 +381,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
},
ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => {
let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else {
return Err(LexicalError {
error: LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()),
location: range.start(),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()),
range.start(),
));
};
unparse_expr(value, buffer)?;
buffer.push('[');
@ -397,10 +397,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
buffer.push_str(attr.as_str());
},
_ => {
return Err(LexicalError {
error: LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()),
location: expr.start(),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()),
expr.start(),
));
}
}
Ok(())
@ -408,10 +408,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
if mode != Mode::Ipython {
return Err(ParseError::User {
error: LexicalError {
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()),
error: LexicalError::new(
LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location,
},
),
});
}
@ -420,10 +420,10 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
2 => IpyEscapeKind::Help2,
_ => {
return Err(ParseError::User {
error: LexicalError {
error: LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()),
error: LexicalError::new(
LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()),
location,
},
),
});
}
};
@ -434,7 +434,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = {
Ok(ast::Stmt::IpyEscapeCommand(
ast::StmtIpyEscapeCommand {
kind,
value,
value: value.into_boxed_str(),
range: (location..end_location).into()
}
))
@ -561,10 +561,10 @@ Pattern: ast::Pattern = {
AsPattern: ast::Pattern = {
<location:@L> <pattern:OrPattern> "as" <name:Identifier> <end_location:@R> =>? {
if name.as_str() == "_" {
Err(LexicalError {
error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()),
location,
})?
))?
} else {
Ok(ast::Pattern::MatchAs(
ast::PatternMatchAs {
@ -1247,10 +1247,10 @@ DoubleStarTypedParameter: ast::Parameter = {
ParameterListStarArgs<ParameterType, StarParameterType, DoubleStarParameterType>: (Option<Box<ast::Parameter>>, Vec<ast::ParameterWithDefault>, Option<Box<ast::Parameter>>) = {
<location:@L> "*" <va:StarParameterType?> <kwonlyargs:("," <ParameterDef<ParameterType>>)*> <kwarg:("," <KwargParameter<DoubleStarParameterType>>)?> =>? {
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
return Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
return Err(LexicalError::new(
LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
location,
})?;
))?;
}
let kwarg = kwarg.flatten();
@ -1364,10 +1364,10 @@ NamedExpression: crate::parser::ParenthesizedExpr = {
LambdaDef: crate::parser::ParenthesizedExpr = {
<location:@L> "lambda" <location_args:@L> <parameters:ParameterList<UntypedParameter, StarUntypedParameter, StarUntypedParameter>?> <end_location_args:@R> ":" <fstring_middle:fstring_middle?> <body:Test<"all">> <end_location:@R> =>? {
if fstring_middle.is_some() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location,
})?;
))?;
}
parameters.as_ref().map(validate_arguments).transpose()?;
@ -1630,10 +1630,10 @@ FStringMiddlePattern: ast::FStringElement = {
FStringReplacementField: ast::FStringElement = {
<location:@L> "{" <value:TestListOrYieldExpr> <debug:"="?> <conversion:FStringConversion?> <format_spec:FStringFormatSpecSuffix?> "}" <end_location:@R> =>? {
if value.expr.is_lambda_expr() && !value.is_parenthesized() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location: value.start(),
})?;
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
value.start(),
))?;
}
let debug_text = debug.map(|_| {
let start_offset = location + "{".text_len();
@ -1677,14 +1677,14 @@ FStringFormatSpec: ast::FStringFormatSpec = {
FStringConversion: (TextSize, ast::ConversionFlag) = {
<location:@L> "!" <name_location:@L> <s:name> =>? {
let conversion = match s.as_str() {
let conversion = match s.as_ref() {
"s" => ast::ConversionFlag::Str,
"r" => ast::ConversionFlag::Repr,
"a" => ast::ConversionFlag::Ascii,
_ => Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
location: name_location,
})?
_ => Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
name_location,
))?
};
Ok((location, conversion))
}
@ -1722,10 +1722,10 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
<location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.expr.is_starred_expr() {
return Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: mid.start(),
})?;
return Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
mid.start(),
))?;
}
Ok(crate::parser::ParenthesizedExpr {
expr: mid.into(),
@ -1751,10 +1751,10 @@ Atom<Goal>: crate::parser::ParenthesizedExpr = {
range: (location..end_location).into(),
}.into(),
"(" <location:@L> "**" <e:Expression<"all">> ")" <end_location:@R> =>? {
Err(LexicalError{
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
location,
}.into())
).into())
},
<location:@L> "{" <e:DictLiteralValues?> "}" <end_location:@R> => {
let (keys, values) = e
@ -2061,19 +2061,19 @@ extern {
float => token::Tok::Float { value: <f64> },
complex => token::Tok::Complex { real: <f64>, imag: <f64> },
string => token::Tok::String {
value: <String>,
value: <Box<str>>,
kind: <StringKind>,
triple_quoted: <bool>
},
fstring_middle => token::Tok::FStringMiddle {
value: <String>,
value: <Box<str>>,
is_raw: <bool>,
triple_quoted: <bool>
},
name => token::Tok::Name { name: <String> },
name => token::Tok::Name { name: <Box<str>> },
ipy_escape_command => token::Tok::IpyEscapeCommand {
kind: <IpyEscapeKind>,
value: <String>
value: <Box<str>>
},
"\n" => token::Tok::Newline,
";" => token::Tok::Semi,

280
crates/ruff_python_parser/src/python.rs
View file

@ -1,5 +1,5 @@
// auto-generated: "lalrpop 0.20.0"
// sha3: aa0540221d25f4eadfc9e043fb4fc631d537b672b8a96785dfec2407e0524b79
// sha3: fd05d84d3b654796ff740a7f905ec0ae8915f43f952428717735481947ab55e1
use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
use crate::{
@ -50,11 +50,11 @@ mod __parse__Top {
Variant0(token::Tok),
Variant1((f64, f64)),
Variant2(f64),
Variant3((String, bool, bool)),
Variant3((Box<str>, bool, bool)),
Variant4(Int),
Variant5((IpyEscapeKind, String)),
Variant6(String),
Variant7((String, StringKind, bool)),
Variant5((IpyEscapeKind, Box<str>)),
Variant6(Box<str>),
Variant7((Box<str>, StringKind, bool)),
Variant8(core::option::Option<token::Tok>),
Variant9(Option<Box<ast::Parameter>>),
Variant10(core::option::Option<Option<Box<ast::Parameter>>>),
@ -151,7 +151,7 @@ mod __parse__Top {
Variant101(ast::TypeParams),
Variant102(core::option::Option<ast::TypeParams>),
Variant103(ast::UnaryOp),
Variant104(core::option::Option<(String, bool, bool)>),
Variant104(core::option::Option<(Box<str>, bool, bool)>),
}
const __ACTION: &[i16] = &[
// State 0
@ -18323,10 +18323,30 @@ mod __parse__Top {
fn __symbol_type_mismatch() -> ! {
panic!("symbol type mismatch")
}
fn __pop_Variant7<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (Box<str>, StringKind, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant3<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (Box<str>, bool, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant5<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (IpyEscapeKind, String), TextSize)
) -> (TextSize, (IpyEscapeKind, Box<str>), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r),
@ -18373,26 +18393,6 @@ mod __parse__Top {
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant7<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (String, StringKind, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant3<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, (String, bool, bool), TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant67<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -18493,6 +18493,16 @@ mod __parse__Top {
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant6<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, Box<str>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant4<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -18523,16 +18533,6 @@ mod __parse__Top {
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant6<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, String, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant69<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -19113,6 +19113,16 @@ mod __parse__Top {
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant104<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant74<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -19133,16 +19143,6 @@ mod __parse__Top {
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant104<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
) -> (TextSize, core::option::Option<(String, bool, bool)>, TextSize)
{
match __symbols.pop() {
Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r),
_ => __symbol_type_mismatch()
}
}
fn __pop_Variant68<
>(
__symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
@ -33541,7 +33541,7 @@ fn __action69<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, n, _): (TextSize, String, TextSize),
(_, n, _): (TextSize, Box<str>, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> ast::Identifier
{
@ -33555,13 +33555,13 @@ fn __action70<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, n, _): (TextSize, String, TextSize),
(_, n, _): (TextSize, Box<str>, TextSize),
(_, n2, _): (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> ast::Identifier
{
{
let mut r = n;
let mut r = String::from(n);
for x in n2 {
r.push('.');
r.push_str(x.1.as_str());
@ -33639,7 +33639,7 @@ fn __action74<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, c, _): (TextSize, (IpyEscapeKind, String), TextSize),
(_, c, _): (TextSize, (IpyEscapeKind, Box<str>), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -33653,10 +33653,10 @@ fn __action74<
}
))
} else {
Err(LexicalError {
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location,
})?
))?
}
}
}
@ -33668,7 +33668,7 @@ fn __action75<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, c, _): (TextSize, (IpyEscapeKind, String), TextSize),
(_, c, _): (TextSize, (IpyEscapeKind, Box<str>), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -33676,10 +33676,10 @@ fn __action75<
if mode == Mode::Ipython {
// This should never occur as the lexer won't allow it.
if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) {
return Err(LexicalError {
error: LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()),
return Err(LexicalError::new(
LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()),
location,
})?;
))?;
}
Ok(ast::ExprIpyEscapeCommand {
kind: c.0,
@ -33687,10 +33687,10 @@ fn __action75<
range: (location..end_location).into()
}.into())
} else {
Err(LexicalError {
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location,
})?
))?
}
}
}
@ -33715,10 +33715,10 @@ fn __action76<
},
ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => {
let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else {
return Err(LexicalError {
error: LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()),
location: range.start(),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()),
range.start(),
));
};
unparse_expr(value, buffer)?;
buffer.push('[');
@ -33731,10 +33731,10 @@ fn __action76<
buffer.push_str(attr.as_str());
},
_ => {
return Err(LexicalError {
error: LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()),
location: expr.start(),
});
return Err(LexicalError::new(
LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()),
expr.start(),
));
}
}
Ok(())
@ -33742,10 +33742,10 @@ fn __action76<
if mode != Mode::Ipython {
return Err(ParseError::User {
error: LexicalError {
error: LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()),
error: LexicalError::new(
LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()),
location,
},
),
});
}
@ -33754,10 +33754,10 @@ fn __action76<
2 => IpyEscapeKind::Help2,
_ => {
return Err(ParseError::User {
error: LexicalError {
error: LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()),
error: LexicalError::new(
LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()),
location,
},
),
});
}
};
@ -33768,7 +33768,7 @@ fn __action76<
Ok(ast::Stmt::IpyEscapeCommand(
ast::StmtIpyEscapeCommand {
kind,
value,
value: value.into_boxed_str(),
range: (location..end_location).into()
}
))
@ -34126,10 +34126,10 @@ fn __action95<
{
{
if name.as_str() == "_" {
Err(LexicalError {
error: LexicalErrorType::OtherError("cannot use '_' as a target".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()),
location,
})?
))?
} else {
Ok(ast::Pattern::MatchAs(
ast::PatternMatchAs {
@ -35910,17 +35910,17 @@ fn __action184<
(_, parameters, _): (TextSize, core::option::Option<ast::Parameters>, TextSize),
(_, end_location_args, _): (TextSize, TextSize, TextSize),
(_, _, _): (TextSize, token::Tok, TextSize),
(_, fstring_middle, _): (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
(_, fstring_middle, _): (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
(_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
{
if fstring_middle.is_some() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location,
})?;
))?;
}
parameters.as_ref().map(validate_arguments).transpose()?;
@ -36363,7 +36363,7 @@ fn __action217<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, string, _): (TextSize, (String, StringKind, bool), TextSize),
(_, string, _): (TextSize, (Box<str>, StringKind, bool), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -36413,7 +36413,7 @@ fn __action220<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, fstring_middle, _): (TextSize, (String, bool, bool), TextSize),
(_, fstring_middle, _): (TextSize, (Box<str>, bool, bool), TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -36441,10 +36441,10 @@ fn __action221<
{
{
if value.expr.is_lambda_expr() && !value.is_parenthesized() {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
location: value.start(),
})?;
return Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::LambdaWithoutParentheses),
value.start(),
))?;
}
let debug_text = debug.map(|_| {
let start_offset = location + "{".text_len();
@ -36514,18 +36514,18 @@ fn __action224<
(_, location, _): (TextSize, TextSize, TextSize),
(_, _, _): (TextSize, token::Tok, TextSize),
(_, name_location, _): (TextSize, TextSize, TextSize),
(_, s, _): (TextSize, String, TextSize),
(_, s, _): (TextSize, Box<str>, TextSize),
) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
{
let conversion = match s.as_str() {
let conversion = match s.as_ref() {
"s" => ast::ConversionFlag::Str,
"r" => ast::ConversionFlag::Repr,
"a" => ast::ConversionFlag::Ascii,
_ => Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
location: name_location,
})?
_ => Err(LexicalError::new(
LexicalErrorType::FStringError(FStringErrorType::InvalidConversionFlag),
name_location,
))?
};
Ok((location, conversion))
}
@ -36899,7 +36899,7 @@ fn __action249<
source_code: &str,
mode: Mode,
(_, location, _): (TextSize, TextSize, TextSize),
(_, s, _): (TextSize, String, TextSize),
(_, s, _): (TextSize, Box<str>, TextSize),
(_, end_location, _): (TextSize, TextSize, TextSize),
) -> ast::Identifier
{
@ -37357,8 +37357,8 @@ fn __action281<
>(
source_code: &str,
mode: Mode,
(_, __0, _): (TextSize, (String, bool, bool), TextSize),
) -> core::option::Option<(String, bool, bool)>
(_, __0, _): (TextSize, (Box<str>, bool, bool), TextSize),
) -> core::option::Option<(Box<str>, bool, bool)>
{
Some(__0)
}
@ -37371,7 +37371,7 @@ fn __action282<
mode: Mode,
__lookbehind: &TextSize,
__lookahead: &TextSize,
) -> core::option::Option<(String, bool, bool)>
) -> core::option::Option<(Box<str>, bool, bool)>
{
None
}
@ -39668,10 +39668,10 @@ fn __action445<
{
{
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
return Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
return Err(LexicalError::new(
LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
location,
})?;
))?;
}
let kwarg = kwarg.flatten();
@ -39793,10 +39793,10 @@ fn __action453<
{
{
if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() {
return Err(LexicalError {
error: LexicalErrorType::OtherError("named arguments must follow bare *".to_string()),
return Err(LexicalError::new(
LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()),
location,
})?;
))?;
}
let kwarg = kwarg.flatten();
@ -41296,10 +41296,10 @@ fn __action554<
{
if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.expr.is_starred_expr() {
return Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: mid.start(),
})?;
return Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
mid.start(),
))?;
}
Ok(crate::parser::ParenthesizedExpr {
expr: mid.into(),
@ -41386,10 +41386,10 @@ fn __action558<
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
{
Err(LexicalError{
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
location,
}.into())
).into())
}
}
@ -41994,10 +41994,10 @@ fn __action596<
{
if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.expr.is_starred_expr() {
return Err(LexicalError{
error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()),
location: mid.start(),
})?;
return Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()),
mid.start(),
))?;
}
Ok(crate::parser::ParenthesizedExpr {
expr: mid.into(),
@ -42084,10 +42084,10 @@ fn __action600<
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
{
Err(LexicalError{
error : LexicalErrorType::OtherError("cannot use double starred expression here".to_string()),
Err(LexicalError::new(
LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()),
location,
}.into())
).into())
}
}
@ -48027,7 +48027,7 @@ fn __action789<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, String, TextSize),
__0: (TextSize, Box<str>, TextSize),
__1: (TextSize, TextSize, TextSize),
) -> ast::Identifier
{
@ -48055,7 +48055,7 @@ fn __action790<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, String, TextSize),
__0: (TextSize, Box<str>, TextSize),
__1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize),
__2: (TextSize, TextSize, TextSize),
) -> ast::Identifier
@ -48408,7 +48408,7 @@ fn __action801<
source_code: &str,
mode: Mode,
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, String, TextSize),
__1: (TextSize, Box<str>, TextSize),
) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
let __start0 = __0.0;
@ -48505,7 +48505,7 @@ fn __action804<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (String, bool, bool), TextSize),
__0: (TextSize, (Box<str>, bool, bool), TextSize),
__1: (TextSize, TextSize, TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -49209,7 +49209,7 @@ fn __action826<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, String, TextSize),
__0: (TextSize, Box<str>, TextSize),
__1: (TextSize, TextSize, TextSize),
) -> ast::Identifier
{
@ -49519,7 +49519,7 @@ fn __action836<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize),
__0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
__1: (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -49547,7 +49547,7 @@ fn __action837<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize),
__0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
__1: (TextSize, TextSize, TextSize),
) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -49609,7 +49609,7 @@ fn __action839<
__1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
__2: (TextSize, TextSize, TextSize),
__3: (TextSize, token::Tok, TextSize),
__4: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__4: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__5: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
__6: (TextSize, TextSize, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
@ -52719,7 +52719,7 @@ fn __action937<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (String, StringKind, bool), TextSize),
__0: (TextSize, (Box<str>, StringKind, bool), TextSize),
__1: (TextSize, TextSize, TextSize),
) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -64211,7 +64211,7 @@ fn __action1304<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, String, TextSize),
__0: (TextSize, Box<str>, TextSize),
) -> ast::Identifier
{
let __start0 = __0.2;
@ -64237,7 +64237,7 @@ fn __action1305<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, String, TextSize),
__0: (TextSize, Box<str>, TextSize),
__1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize),
) -> ast::Identifier
{
@ -64527,7 +64527,7 @@ fn __action1315<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (String, bool, bool), TextSize),
__0: (TextSize, (Box<str>, bool, bool), TextSize),
) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
let __start0 = __0.2;
@ -65035,7 +65035,7 @@ fn __action1333<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, String, TextSize),
__0: (TextSize, Box<str>, TextSize),
) -> ast::Identifier
{
let __start0 = __0.2;
@ -65347,7 +65347,7 @@ fn __action1344<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize),
__0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
let __start0 = __0.2;
@ -65373,7 +65373,7 @@ fn __action1345<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (IpyEscapeKind, String), TextSize),
__0: (TextSize, (IpyEscapeKind, Box<str>), TextSize),
) -> Result<ast::Stmt,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
let __start0 = __0.2;
@ -65430,7 +65430,7 @@ fn __action1347<
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
__2: (TextSize, token::Tok, TextSize),
__3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__3: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -69997,7 +69997,7 @@ fn __action1494<
>(
source_code: &str,
mode: Mode,
__0: (TextSize, (String, StringKind, bool), TextSize),
__0: (TextSize, (Box<str>, StringKind, bool), TextSize),
) -> Result<StringType,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
let __start0 = __0.2;
@ -77662,7 +77662,7 @@ fn __action1727<
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, ast::Parameters, TextSize),
__2: (TextSize, token::Tok, TextSize),
__3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__3: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -77693,7 +77693,7 @@ fn __action1728<
mode: Mode,
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, token::Tok, TextSize),
__2: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
__2: (TextSize, core::option::Option<(Box<str>, bool, bool)>, TextSize),
__3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -79598,7 +79598,7 @@ fn __action1785<
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, ast::Parameters, TextSize),
__2: (TextSize, token::Tok, TextSize),
__3: (TextSize, (String, bool, bool), TextSize),
__3: (TextSize, (Box<str>, bool, bool), TextSize),
__4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{
@ -79661,7 +79661,7 @@ fn __action1787<
mode: Mode,
__0: (TextSize, token::Tok, TextSize),
__1: (TextSize, token::Tok, TextSize),
__2: (TextSize, (String, bool, bool), TextSize),
__2: (TextSize, (Box<str>, bool, bool), TextSize),
__3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
{

2
crates/ruff_python_parser/src/soft_keywords.rs
View file

@ -203,7 +203,7 @@ fn soft_to_name(tok: &Tok) -> Tok {
_ => unreachable!("other tokens never reach here"),
};
Tok::Name {
name: name.to_owned(),
name: name.to_string().into_boxed_str(),
}
}

50
crates/ruff_python_parser/src/string.rs
View file

@ -151,10 +151,10 @@ impl<'a> StringParser<'a> {
fn parse_escaped_char(&mut self, string: &mut String) -> Result<(), LexicalError> {
let Some(first_char) = self.next_char() else {
return Err(LexicalError {
error: LexicalErrorType::StringError,
location: self.get_pos(),
});
return Err(LexicalError::new(
LexicalErrorType::StringError,
self.get_pos(),
));
};
let new_char = match first_char {
@ -184,12 +184,14 @@ impl<'a> StringParser<'a> {
}
_ => {
if self.kind.is_any_bytes() && !first_char.is_ascii() {
return Err(LexicalError {
error: LexicalErrorType::OtherError(
"bytes can only contain ASCII literal characters".to_owned(),
return Err(LexicalError::new(
LexicalErrorType::OtherError(
"bytes can only contain ASCII literal characters"
.to_string()
.into_boxed_str(),
),
location: self.get_pos(),
});
self.get_pos(),
));
}
string.push('\\');
@ -257,7 +259,9 @@ impl<'a> StringParser<'a> {
if !ch.is_ascii() {
return Err(LexicalError::new(
LexicalErrorType::OtherError(
"bytes can only contain ASCII literal characters".to_string(),
"bytes can only contain ASCII literal characters"
.to_string()
.into_boxed_str(),
),
self.get_pos(),
));
@ -291,7 +295,7 @@ impl<'a> StringParser<'a> {
}
}
Ok(StringType::Str(ast::StringLiteral {
value,
value: value.into_boxed_str(),
unicode: self.kind.is_unicode(),
range: self.range,
}))
@ -354,12 +358,14 @@ pub(crate) fn concatenated_strings(
let has_bytes = byte_literal_count > 0;
if has_bytes && byte_literal_count < strings.len() {
return Err(LexicalError {
error: LexicalErrorType::OtherError(
"cannot mix bytes and nonbytes literals".to_owned(),
return Err(LexicalError::new(
LexicalErrorType::OtherError(
"cannot mix bytes and nonbytes literals"
.to_string()
.into_boxed_str(),
),
location: range.start(),
});
range.start(),
));
}
if has_bytes {
@ -418,15 +424,12 @@ struct FStringError {
impl From<FStringError> for LexicalError {
fn from(err: FStringError) -> Self {
LexicalError {
error: LexicalErrorType::FStringError(err.error),
location: err.location,
}
LexicalError::new(LexicalErrorType::FStringError(err.error), err.location)
}
}
/// Represents the different types of errors that can occur during parsing of an f-string.
#[derive(Debug, Clone, PartialEq)]
#[derive(Copy, Debug, Clone, PartialEq)]
pub enum FStringErrorType {
/// Expected a right brace after an opened left brace.
UnclosedLbrace,
@ -466,10 +469,7 @@ impl std::fmt::Display for FStringErrorType {
impl From<FStringError> for crate::parser::LalrpopError<TextSize, Tok, LexicalError> {
fn from(err: FStringError) -> Self {
lalrpop_util::ParseError::User {
error: LexicalError {
error: LexicalErrorType::FStringError(err.error),
location: err.location,
},
error: LexicalError::new(LexicalErrorType::FStringError(err.error), err.location),
}
}
}

21
crates/ruff_python_parser/src/token.rs
View file

@ -16,7 +16,7 @@ pub enum Tok {
/// Token value for a name, commonly known as an identifier.
Name {
/// The name value.
name: String,
name: Box<str>,
},
/// Token value for an integer.
Int {
@ -38,7 +38,7 @@ pub enum Tok {
/// Token value for a string.
String {
/// The string value.
value: String,
value: Box<str>,
/// The kind of string.
kind: StringKind,
/// Whether the string is triple quoted.
@ -51,7 +51,7 @@ pub enum Tok {
/// part of the expression part and isn't an opening or closing brace.
FStringMiddle {
/// The string value.
value: String,
value: Box<str>,
/// Whether the string is raw or not.
is_raw: bool,
/// Whether the string is triple quoted.
@ -63,12 +63,12 @@ pub enum Tok {
/// only when the mode is [`Mode::Ipython`].
IpyEscapeCommand {
/// The magic command value.
value: String,
value: Box<str>,
/// The kind of magic command.
kind: IpyEscapeKind,
},
/// Token value for a comment. These are filtered out of the token stream prior to parsing.
Comment(String),
Comment(Box<str>),
/// Token value for a newline.
Newline,
/// Token value for a newline that is not a logical line break. These are filtered out of
@ -912,3 +912,14 @@ impl From<&Tok> for TokenKind {
Self::from_token(value)
}
}
#[cfg(target_pointer_width = "64")]
mod sizes {
use crate::lexer::{LexicalError, LexicalErrorType};
use crate::Tok;
use static_assertions::assert_eq_size;
assert_eq_size!(Tok, [u8; 24]);
assert_eq_size!(LexicalErrorType, [u8; 24]);
assert_eq_size!(Result<Tok, LexicalError>, [u8; 32]);
}