mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-09 22:25:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Try ThinVec crate to reduce AST size

Browse source
This commit is contained in:
Micha Reiser 2023-05-14 22:22:29 +02:00
parent 718354673e
commit c4847adf2f
No known key found for this signature in database
12 changed files with 33288 additions and 47657 deletions
Download patch file Download diff file Expand all files Collapse all files

1
ast/Cargo.toml
View file

@ -23,3 +23,4 @@ rustpython-literal = { workspace = true, optional = true }
is-macro = { workspace = true } is-macro = { workspace = true }
num-bigint = { workspace = true } num-bigint = { workspace = true }
static_assertions = "1.1.0" static_assertions = "1.1.0"
thin-vec = "0.2.12"

2
ast/asdl_rs.py
View file

@ -372,7 +372,7 @@ class StructVisitor(EmitVisitor):
): ):
typ = f"Option<{typ}>" typ = f"Option<{typ}>"
if field.seq: if field.seq:
typ = f"Vec<{typ}>" typ = f"thin_vec::ThinVec<{typ}>"
if typ == "Int": if typ == "Int":
typ = BUILTIN_INT_NAMES.get(field.name, typ) typ = BUILTIN_INT_NAMES.get(field.name, typ)
name = rust_field(field.name) name = rust_field(field.name)

5
ast/src/fold_helpers.rs
View file

@ -1,4 +1,5 @@
use crate::{builtin, fold::Fold}; use crate::{builtin, fold::Fold};
use thin_vec::ThinVec;
pub trait Foldable<T, U> { pub trait Foldable<T, U> {
type Mapped; type Mapped;
@ -8,11 +9,11 @@ pub trait Foldable<T, U> {
) -> Result<Self::Mapped, F::Error>; ) -> Result<Self::Mapped, F::Error>;
} }
impl<T, U, X> Foldable<T, U> for Vec<X> impl<T, U, X> Foldable<T, U> for ThinVec<X>
where where
X: Foldable<T, U>, X: Foldable<T, U>,
{ {
type Mapped = Vec<X::Mapped>; type Mapped = ThinVec<X::Mapped>;
fn fold<F: Fold<T, TargetU = U> + ?Sized>( fn fold<F: Fold<T, TargetU = U> + ?Sized>(
self, self,
folder: &mut F, folder: &mut F,

138
ast/src/gen/generic.rs
View file

@ -4,8 +4,8 @@ use crate::text_size::TextRange;
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ModModule<R = TextRange> { pub struct ModModule<R = TextRange> {
pub range: crate::ranged::OptionalRange<R>, pub range: crate::ranged::OptionalRange<R>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub type_ignores: Vec<TypeIgnore<R>>, pub type_ignores: thin_vec::ThinVec<TypeIgnore<R>>,
} }
impl<R> From<ModModule<R>> for Mod<R> { impl<R> From<ModModule<R>> for Mod<R> {
@ -17,7 +17,7 @@ impl<R> From<ModModule<R>> for Mod<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ModInteractive<R = TextRange> { pub struct ModInteractive<R = TextRange> {
pub range: crate::ranged::OptionalRange<R>, pub range: crate::ranged::OptionalRange<R>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
} }
impl<R> From<ModInteractive<R>> for Mod<R> { impl<R> From<ModInteractive<R>> for Mod<R> {
@ -41,7 +41,7 @@ impl<R> From<ModExpression<R>> for Mod<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ModFunctionType<R = TextRange> { pub struct ModFunctionType<R = TextRange> {
pub range: crate::ranged::OptionalRange<R>, pub range: crate::ranged::OptionalRange<R>,
pub argtypes: Vec<Expr<R>>, pub argtypes: thin_vec::ThinVec<Expr<R>>,
pub returns: Box<Expr<R>>, pub returns: Box<Expr<R>>,
} }
@ -64,8 +64,8 @@ pub struct StmtFunctionDef<R = TextRange> {
pub range: R, pub range: R,
pub name: Identifier, pub name: Identifier,
pub args: Box<Arguments<R>>, pub args: Box<Arguments<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub decorator_list: Vec<Expr<R>>, pub decorator_list: thin_vec::ThinVec<Expr<R>>,
pub returns: Option<Box<Expr<R>>>, pub returns: Option<Box<Expr<R>>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -81,8 +81,8 @@ pub struct StmtAsyncFunctionDef<R = TextRange> {
pub range: R, pub range: R,
pub name: Identifier, pub name: Identifier,
pub args: Box<Arguments<R>>, pub args: Box<Arguments<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub decorator_list: Vec<Expr<R>>, pub decorator_list: thin_vec::ThinVec<Expr<R>>,
pub returns: Option<Box<Expr<R>>>, pub returns: Option<Box<Expr<R>>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -97,10 +97,10 @@ impl<R> From<StmtAsyncFunctionDef<R>> for Stmt<R> {
pub struct StmtClassDef<R = TextRange> { pub struct StmtClassDef<R = TextRange> {
pub range: R, pub range: R,
pub name: Identifier, pub name: Identifier,
pub bases: Vec<Expr<R>>, pub bases: thin_vec::ThinVec<Expr<R>>,
pub keywords: Vec<Keyword<R>>, pub keywords: thin_vec::ThinVec<Keyword<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub decorator_list: Vec<Expr<R>>, pub decorator_list: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<StmtClassDef<R>> for Stmt<R> { impl<R> From<StmtClassDef<R>> for Stmt<R> {
@ -124,7 +124,7 @@ impl<R> From<StmtReturn<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtDelete<R = TextRange> { pub struct StmtDelete<R = TextRange> {
pub range: R, pub range: R,
pub targets: Vec<Expr<R>>, pub targets: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<StmtDelete<R>> for Stmt<R> { impl<R> From<StmtDelete<R>> for Stmt<R> {
@ -136,7 +136,7 @@ impl<R> From<StmtDelete<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtAssign<R = TextRange> { pub struct StmtAssign<R = TextRange> {
pub range: R, pub range: R,
pub targets: Vec<Expr<R>>, pub targets: thin_vec::ThinVec<Expr<R>>,
pub value: Box<Expr<R>>, pub value: Box<Expr<R>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -181,8 +181,8 @@ pub struct StmtFor<R = TextRange> {
pub range: R, pub range: R,
pub target: Box<Expr<R>>, pub target: Box<Expr<R>>,
pub iter: Box<Expr<R>>, pub iter: Box<Expr<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub orelse: Vec<Stmt<R>>, pub orelse: thin_vec::ThinVec<Stmt<R>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -197,8 +197,8 @@ pub struct StmtAsyncFor<R = TextRange> {
pub range: R, pub range: R,
pub target: Box<Expr<R>>, pub target: Box<Expr<R>>,
pub iter: Box<Expr<R>>, pub iter: Box<Expr<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub orelse: Vec<Stmt<R>>, pub orelse: thin_vec::ThinVec<Stmt<R>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -212,8 +212,8 @@ impl<R> From<StmtAsyncFor<R>> for Stmt<R> {
pub struct StmtWhile<R = TextRange> { pub struct StmtWhile<R = TextRange> {
pub range: R, pub range: R,
pub test: Box<Expr<R>>, pub test: Box<Expr<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub orelse: Vec<Stmt<R>>, pub orelse: thin_vec::ThinVec<Stmt<R>>,
} }
impl<R> From<StmtWhile<R>> for Stmt<R> { impl<R> From<StmtWhile<R>> for Stmt<R> {
@ -226,8 +226,8 @@ impl<R> From<StmtWhile<R>> for Stmt<R> {
pub struct StmtIf<R = TextRange> { pub struct StmtIf<R = TextRange> {
pub range: R, pub range: R,
pub test: Box<Expr<R>>, pub test: Box<Expr<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub orelse: Vec<Stmt<R>>, pub orelse: thin_vec::ThinVec<Stmt<R>>,
} }
impl<R> From<StmtIf<R>> for Stmt<R> { impl<R> From<StmtIf<R>> for Stmt<R> {
@ -239,8 +239,8 @@ impl<R> From<StmtIf<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtWith<R = TextRange> { pub struct StmtWith<R = TextRange> {
pub range: R, pub range: R,
pub items: Vec<Withitem<R>>, pub items: thin_vec::ThinVec<Withitem<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -253,8 +253,8 @@ impl<R> From<StmtWith<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtAsyncWith<R = TextRange> { pub struct StmtAsyncWith<R = TextRange> {
pub range: R, pub range: R,
pub items: Vec<Withitem<R>>, pub items: thin_vec::ThinVec<Withitem<R>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub type_comment: Option<String>, pub type_comment: Option<String>,
} }
@ -268,7 +268,7 @@ impl<R> From<StmtAsyncWith<R>> for Stmt<R> {
pub struct StmtMatch<R = TextRange> { pub struct StmtMatch<R = TextRange> {
pub range: R, pub range: R,
pub subject: Box<Expr<R>>, pub subject: Box<Expr<R>>,
pub cases: Vec<MatchCase<R>>, pub cases: thin_vec::ThinVec<MatchCase<R>>,
} }
impl<R> From<StmtMatch<R>> for Stmt<R> { impl<R> From<StmtMatch<R>> for Stmt<R> {
@ -293,10 +293,10 @@ impl<R> From<StmtRaise<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtTry<R = TextRange> { pub struct StmtTry<R = TextRange> {
pub range: R, pub range: R,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub handlers: Vec<Excepthandler<R>>, pub handlers: thin_vec::ThinVec<Excepthandler<R>>,
pub orelse: Vec<Stmt<R>>, pub orelse: thin_vec::ThinVec<Stmt<R>>,
pub finalbody: Vec<Stmt<R>>, pub finalbody: thin_vec::ThinVec<Stmt<R>>,
} }
impl<R> From<StmtTry<R>> for Stmt<R> { impl<R> From<StmtTry<R>> for Stmt<R> {
@ -308,10 +308,10 @@ impl<R> From<StmtTry<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtTryStar<R = TextRange> { pub struct StmtTryStar<R = TextRange> {
pub range: R, pub range: R,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub handlers: Vec<Excepthandler<R>>, pub handlers: thin_vec::ThinVec<Excepthandler<R>>,
pub orelse: Vec<Stmt<R>>, pub orelse: thin_vec::ThinVec<Stmt<R>>,
pub finalbody: Vec<Stmt<R>>, pub finalbody: thin_vec::ThinVec<Stmt<R>>,
} }
impl<R> From<StmtTryStar<R>> for Stmt<R> { impl<R> From<StmtTryStar<R>> for Stmt<R> {
@ -336,7 +336,7 @@ impl<R> From<StmtAssert<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtImport<R = TextRange> { pub struct StmtImport<R = TextRange> {
pub range: R, pub range: R,
pub names: Vec<Alias<R>>, pub names: thin_vec::ThinVec<Alias<R>>,
} }
impl<R> From<StmtImport<R>> for Stmt<R> { impl<R> From<StmtImport<R>> for Stmt<R> {
@ -349,7 +349,7 @@ impl<R> From<StmtImport<R>> for Stmt<R> {
pub struct StmtImportFrom<R = TextRange> { pub struct StmtImportFrom<R = TextRange> {
pub range: R, pub range: R,
pub module: Option<Identifier>, pub module: Option<Identifier>,
pub names: Vec<Alias<R>>, pub names: thin_vec::ThinVec<Alias<R>>,
pub level: Option<Int>, pub level: Option<Int>,
} }
@ -362,7 +362,7 @@ impl<R> From<StmtImportFrom<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtGlobal<R = TextRange> { pub struct StmtGlobal<R = TextRange> {
pub range: R, pub range: R,
pub names: Vec<Identifier>, pub names: thin_vec::ThinVec<Identifier>,
} }
impl<R> From<StmtGlobal<R>> for Stmt<R> { impl<R> From<StmtGlobal<R>> for Stmt<R> {
@ -374,7 +374,7 @@ impl<R> From<StmtGlobal<R>> for Stmt<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct StmtNonlocal<R = TextRange> { pub struct StmtNonlocal<R = TextRange> {
pub range: R, pub range: R,
pub names: Vec<Identifier>, pub names: thin_vec::ThinVec<Identifier>,
} }
impl<R> From<StmtNonlocal<R>> for Stmt<R> { impl<R> From<StmtNonlocal<R>> for Stmt<R> {
@ -490,7 +490,7 @@ pub enum Stmt<R = TextRange> {
pub struct ExprBoolOp<R = TextRange> { pub struct ExprBoolOp<R = TextRange> {
pub range: R, pub range: R,
pub op: Boolop, pub op: Boolop,
pub values: Vec<Expr<R>>, pub values: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<ExprBoolOp<R>> for Expr<R> { impl<R> From<ExprBoolOp<R>> for Expr<R> {
@ -569,8 +569,8 @@ impl<R> From<ExprIfExp<R>> for Expr<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ExprDict<R = TextRange> { pub struct ExprDict<R = TextRange> {
pub range: R, pub range: R,
pub keys: Vec<Option<Expr<R>>>, pub keys: thin_vec::ThinVec<Option<Expr<R>>>,
pub values: Vec<Expr<R>>, pub values: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<ExprDict<R>> for Expr<R> { impl<R> From<ExprDict<R>> for Expr<R> {
@ -582,7 +582,7 @@ impl<R> From<ExprDict<R>> for Expr<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ExprSet<R = TextRange> { pub struct ExprSet<R = TextRange> {
pub range: R, pub range: R,
pub elts: Vec<Expr<R>>, pub elts: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<ExprSet<R>> for Expr<R> { impl<R> From<ExprSet<R>> for Expr<R> {
@ -595,7 +595,7 @@ impl<R> From<ExprSet<R>> for Expr<R> {
pub struct ExprListComp<R = TextRange> { pub struct ExprListComp<R = TextRange> {
pub range: R, pub range: R,
pub elt: Box<Expr<R>>, pub elt: Box<Expr<R>>,
pub generators: Vec<Comprehension<R>>, pub generators: thin_vec::ThinVec<Comprehension<R>>,
} }
impl<R> From<ExprListComp<R>> for Expr<R> { impl<R> From<ExprListComp<R>> for Expr<R> {
@ -608,7 +608,7 @@ impl<R> From<ExprListComp<R>> for Expr<R> {
pub struct ExprSetComp<R = TextRange> { pub struct ExprSetComp<R = TextRange> {
pub range: R, pub range: R,
pub elt: Box<Expr<R>>, pub elt: Box<Expr<R>>,
pub generators: Vec<Comprehension<R>>, pub generators: thin_vec::ThinVec<Comprehension<R>>,
} }
impl<R> From<ExprSetComp<R>> for Expr<R> { impl<R> From<ExprSetComp<R>> for Expr<R> {
@ -622,7 +622,7 @@ pub struct ExprDictComp<R = TextRange> {
pub range: R, pub range: R,
pub key: Box<Expr<R>>, pub key: Box<Expr<R>>,
pub value: Box<Expr<R>>, pub value: Box<Expr<R>>,
pub generators: Vec<Comprehension<R>>, pub generators: thin_vec::ThinVec<Comprehension<R>>,
} }
impl<R> From<ExprDictComp<R>> for Expr<R> { impl<R> From<ExprDictComp<R>> for Expr<R> {
@ -635,7 +635,7 @@ impl<R> From<ExprDictComp<R>> for Expr<R> {
pub struct ExprGeneratorExp<R = TextRange> { pub struct ExprGeneratorExp<R = TextRange> {
pub range: R, pub range: R,
pub elt: Box<Expr<R>>, pub elt: Box<Expr<R>>,
pub generators: Vec<Comprehension<R>>, pub generators: thin_vec::ThinVec<Comprehension<R>>,
} }
impl<R> From<ExprGeneratorExp<R>> for Expr<R> { impl<R> From<ExprGeneratorExp<R>> for Expr<R> {
@ -684,8 +684,8 @@ impl<R> From<ExprYieldFrom<R>> for Expr<R> {
pub struct ExprCompare<R = TextRange> { pub struct ExprCompare<R = TextRange> {
pub range: R, pub range: R,
pub left: Box<Expr<R>>, pub left: Box<Expr<R>>,
pub ops: Vec<Cmpop>, pub ops: thin_vec::ThinVec<Cmpop>,
pub comparators: Vec<Expr<R>>, pub comparators: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<ExprCompare<R>> for Expr<R> { impl<R> From<ExprCompare<R>> for Expr<R> {
@ -698,8 +698,8 @@ impl<R> From<ExprCompare<R>> for Expr<R> {
pub struct ExprCall<R = TextRange> { pub struct ExprCall<R = TextRange> {
pub range: R, pub range: R,
pub func: Box<Expr<R>>, pub func: Box<Expr<R>>,
pub args: Vec<Expr<R>>, pub args: thin_vec::ThinVec<Expr<R>>,
pub keywords: Vec<Keyword<R>>, pub keywords: thin_vec::ThinVec<Keyword<R>>,
} }
impl<R> From<ExprCall<R>> for Expr<R> { impl<R> From<ExprCall<R>> for Expr<R> {
@ -725,7 +725,7 @@ impl<R> From<ExprFormattedValue<R>> for Expr<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ExprJoinedStr<R = TextRange> { pub struct ExprJoinedStr<R = TextRange> {
pub range: R, pub range: R,
pub values: Vec<Expr<R>>, pub values: thin_vec::ThinVec<Expr<R>>,
} }
impl<R> From<ExprJoinedStr<R>> for Expr<R> { impl<R> From<ExprJoinedStr<R>> for Expr<R> {
@ -804,7 +804,7 @@ impl<R> From<ExprName<R>> for Expr<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ExprList<R = TextRange> { pub struct ExprList<R = TextRange> {
pub range: R, pub range: R,
pub elts: Vec<Expr<R>>, pub elts: thin_vec::ThinVec<Expr<R>>,
pub ctx: ExprContext, pub ctx: ExprContext,
} }
@ -817,7 +817,7 @@ impl<R> From<ExprList<R>> for Expr<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct ExprTuple<R = TextRange> { pub struct ExprTuple<R = TextRange> {
pub range: R, pub range: R,
pub elts: Vec<Expr<R>>, pub elts: thin_vec::ThinVec<Expr<R>>,
pub ctx: ExprContext, pub ctx: ExprContext,
} }
@ -955,7 +955,7 @@ pub enum Cmpop {
pub struct Comprehension<R = TextRange> { pub struct Comprehension<R = TextRange> {
pub target: Expr<R>, pub target: Expr<R>,
pub iter: Expr<R>, pub iter: Expr<R>,
pub ifs: Vec<Expr<R>>, pub ifs: thin_vec::ThinVec<Expr<R>>,
pub is_async: bool, pub is_async: bool,
pub range: crate::ranged::OptionalRange<R>, pub range: crate::ranged::OptionalRange<R>,
} }
@ -965,7 +965,7 @@ pub struct ExcepthandlerExceptHandler<R = TextRange> {
pub range: R, pub range: R,
pub type_: Option<Box<Expr<R>>>, pub type_: Option<Box<Expr<R>>>,
pub name: Option<Identifier>, pub name: Option<Identifier>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
} }
impl<R> From<ExcepthandlerExceptHandler<R>> for Excepthandler<R> { impl<R> From<ExcepthandlerExceptHandler<R>> for Excepthandler<R> {
@ -981,13 +981,13 @@ pub enum Excepthandler<R = TextRange> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct Arguments<R = TextRange> { pub struct Arguments<R = TextRange> {
pub posonlyargs: Vec<Arg<R>>, pub posonlyargs: thin_vec::ThinVec<Arg<R>>,
pub args: Vec<Arg<R>>, pub args: thin_vec::ThinVec<Arg<R>>,
pub vararg: Option<Box<Arg<R>>>, pub vararg: Option<Box<Arg<R>>>,
pub kwonlyargs: Vec<Arg<R>>, pub kwonlyargs: thin_vec::ThinVec<Arg<R>>,
pub kw_defaults: Vec<Expr<R>>, pub kw_defaults: thin_vec::ThinVec<Expr<R>>,
pub kwarg: Option<Box<Arg<R>>>, pub kwarg: Option<Box<Arg<R>>>,
pub defaults: Vec<Expr<R>>, pub defaults: thin_vec::ThinVec<Expr<R>>,
pub range: crate::ranged::OptionalRange<R>, pub range: crate::ranged::OptionalRange<R>,
} }
@ -1024,7 +1024,7 @@ pub struct Withitem<R = TextRange> {
pub struct MatchCase<R = TextRange> { pub struct MatchCase<R = TextRange> {
pub pattern: Pattern<R>, pub pattern: Pattern<R>,
pub guard: Option<Box<Expr<R>>>, pub guard: Option<Box<Expr<R>>>,
pub body: Vec<Stmt<R>>, pub body: thin_vec::ThinVec<Stmt<R>>,
pub range: crate::ranged::OptionalRange<R>, pub range: crate::ranged::OptionalRange<R>,
} }
@ -1055,7 +1055,7 @@ impl<R> From<PatternMatchSingleton<R>> for Pattern<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct PatternMatchSequence<R = TextRange> { pub struct PatternMatchSequence<R = TextRange> {
pub range: R, pub range: R,
pub patterns: Vec<Pattern<R>>, pub patterns: thin_vec::ThinVec<Pattern<R>>,
} }
impl<R> From<PatternMatchSequence<R>> for Pattern<R> { impl<R> From<PatternMatchSequence<R>> for Pattern<R> {
@ -1067,8 +1067,8 @@ impl<R> From<PatternMatchSequence<R>> for Pattern<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct PatternMatchMapping<R = TextRange> { pub struct PatternMatchMapping<R = TextRange> {
pub range: R, pub range: R,
pub keys: Vec<Expr<R>>, pub keys: thin_vec::ThinVec<Expr<R>>,
pub patterns: Vec<Pattern<R>>, pub patterns: thin_vec::ThinVec<Pattern<R>>,
pub rest: Option<Identifier>, pub rest: Option<Identifier>,
} }
@ -1082,9 +1082,9 @@ impl<R> From<PatternMatchMapping<R>> for Pattern<R> {
pub struct PatternMatchClass<R = TextRange> { pub struct PatternMatchClass<R = TextRange> {
pub range: R, pub range: R,
pub cls: Box<Expr<R>>, pub cls: Box<Expr<R>>,
pub patterns: Vec<Pattern<R>>, pub patterns: thin_vec::ThinVec<Pattern<R>>,
pub kwd_attrs: Vec<Identifier>, pub kwd_attrs: thin_vec::ThinVec<Identifier>,
pub kwd_patterns: Vec<Pattern<R>>, pub kwd_patterns: thin_vec::ThinVec<Pattern<R>>,
} }
impl<R> From<PatternMatchClass<R>> for Pattern<R> { impl<R> From<PatternMatchClass<R>> for Pattern<R> {
@ -1121,7 +1121,7 @@ impl<R> From<PatternMatchAs<R>> for Pattern<R> {
#[derive(Clone, Debug, PartialEq)] #[derive(Clone, Debug, PartialEq)]
pub struct PatternMatchOr<R = TextRange> { pub struct PatternMatchOr<R = TextRange> {
pub range: R, pub range: R,
pub patterns: Vec<Pattern<R>>, pub patterns: thin_vec::ThinVec<Pattern<R>>,
} }
impl<R> From<PatternMatchOr<R>> for Pattern<R> { impl<R> From<PatternMatchOr<R>> for Pattern<R> {

6
ast/src/impls.rs
View file

@ -57,8 +57,8 @@ impl<R> Expr<R> {
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
static_assertions::assert_eq_size!(crate::Expr, [u8; 72]); static_assertions::assert_eq_size!(crate::Expr, [u8; 72]);
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
static_assertions::assert_eq_size!(crate::Stmt, [u8; 136]); static_assertions::assert_eq_size!(crate::Stmt, [u8; 96]);
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
static_assertions::assert_eq_size!(crate::Pattern, [u8; 96]); static_assertions::assert_eq_size!(crate::Pattern, [u8; 56]);
#[cfg(target_arch = "x86_64")] #[cfg(target_arch = "x86_64")]
static_assertions::assert_eq_size!(crate::Excepthandler, [u8; 64]); static_assertions::assert_eq_size!(crate::Excepthandler, [u8; 48]);

4
ast/src/located.rs
View file

@ -1,5 +1,7 @@
#![allow(clippy::derive_partial_eq_without_eq)] #![allow(clippy::derive_partial_eq_without_eq)]
use crate::source_code::{SourceLocation, SourceRange}; use crate::source_code::{SourceLocation, SourceRange};
use thin_vec::ThinVec;
pub trait Located { pub trait Located {
fn range(&self) -> SourceRange; fn range(&self) -> SourceRange;
@ -13,7 +15,7 @@ pub trait Located {
} }
} }
pub type Suite = Vec<Stmt>; pub type Suite = ThinVec<Stmt>;
pub use crate::builtin::*; pub use crate::builtin::*;
include!("gen/located.rs"); include!("gen/located.rs");

3
ast/src/ranged.rs
View file

@ -1,6 +1,7 @@
use crate::text_size::{TextRange, TextSize}; use crate::text_size::{TextRange, TextSize};
use std::fmt::{Debug, Display, Formatter}; use std::fmt::{Debug, Display, Formatter};
use std::marker::PhantomData; use std::marker::PhantomData;
use thin_vec::ThinVec;
pub use crate::builtin::*; pub use crate::builtin::*;
use crate::Stmt; use crate::Stmt;
@ -17,7 +18,7 @@ pub trait Ranged {
} }
} }
pub type Suite<R = TextRange> = Vec<Stmt<R>>; pub type Suite<R = TextRange> = ThinVec<Stmt<R>>;
#[cfg(feature = "all-nodes-with-ranges")] #[cfg(feature = "all-nodes-with-ranges")]
pub type OptionalRange<R> = R; pub type OptionalRange<R> = R;

1
parser/Cargo.toml
View file

@ -36,6 +36,7 @@ lalrpop-util = { version = "0.20.0", default-features = false }
phf = "0.11.1" phf = "0.11.1"
rustc-hash = "1.1.0" rustc-hash = "1.1.0"
serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] } serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
thin-vec = "0.2.12"
[dev-dependencies] [dev-dependencies]
insta = { workspace = true } insta = { workspace = true }

23
parser/src/function.rs
View file

@ -8,13 +8,14 @@ use crate::{
}; };
use rustc_hash::FxHashSet; use rustc_hash::FxHashSet;
use rustpython_ast::Ranged; use rustpython_ast::Ranged;
use thin_vec::{thin_vec, ThinVec};
pub(crate) struct ArgumentList { pub(crate) struct ArgumentList {
pub args: Vec<ast::Expr>, pub args: ThinVec<ast::Expr>,
pub keywords: Vec<ast::Keyword>, pub keywords: ThinVec<ast::Keyword>,
} }
type ParameterDefs = (Vec<ast::Arg>, Vec<ast::Arg>, Vec<ast::Expr>); type ParameterDefs = (ThinVec<ast::Arg>, ThinVec<ast::Arg>, ThinVec<ast::Expr>);
type ParameterDef = (ast::Arg, Option<ast::Expr>); type ParameterDef = (ast::Arg, Option<ast::Expr>);
// Perform validation of function/lambda arguments in a function definition. // Perform validation of function/lambda arguments in a function definition.
@ -53,11 +54,11 @@ pub(crate) fn validate_arguments(
// Parse parameters as supplied during a function/lambda *definition*. // Parse parameters as supplied during a function/lambda *definition*.
pub(crate) fn parse_params( pub(crate) fn parse_params(
params: (Vec<ParameterDef>, Vec<ParameterDef>), params: (ThinVec<ParameterDef>, ThinVec<ParameterDef>),
) -> Result<ParameterDefs, LexicalError> { ) -> Result<ParameterDefs, LexicalError> {
let mut pos_only = Vec::with_capacity(params.0.len()); let mut pos_only = ThinVec::with_capacity(params.0.len());
let mut names = Vec::with_capacity(params.1.len()); let mut names = ThinVec::with_capacity(params.1.len());
let mut defaults = vec![]; let mut defaults = thin_vec![];
let mut try_default = |name: &ast::Arg, default| { let mut try_default = |name: &ast::Arg, default| {
if let Some(default) = default { if let Some(default) = default {
@ -92,9 +93,11 @@ type FunctionArgument = (
); );
// Parse arguments as supplied during a function/lambda *call*. // Parse arguments as supplied during a function/lambda *call*.
pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentList, LexicalError> { pub(crate) fn parse_args(
let mut args = vec![]; func_args: ThinVec<FunctionArgument>,
let mut keywords = vec![]; ) -> Result<ArgumentList, LexicalError> {
let mut args = thin_vec![];
let mut keywords = thin_vec![];
let mut keyword_names = let mut keyword_names =
FxHashSet::with_capacity_and_hasher(func_args.len(), Default::default()); FxHashSet::with_capacity_and_hasher(func_args.len(), Default::default());

245
parser/src/python.lalrpop
View file

@ -13,6 +13,7 @@ use crate::{
text_size::TextSize, parser::optional_range text_size::TextSize, parser::optional_range
}; };
use num_bigint::BigInt; use num_bigint::BigInt;
use thin_vec::{thin_vec, ThinVec};
grammar; grammar;
@ -20,38 +21,36 @@ grammar;
// For each public entry point, a full parse table is generated. // For each public entry point, a full parse table is generated.
// By having only a single pub function, we reduce this to one. // By having only a single pub function, we reduce this to one.
pub Top: ast::Mod = { pub Top: ast::Mod = {
<start:@L> StartModule <body:Program> <end:@R> => ast::ModModule { body, type_ignores: vec![], range: optional_range(start, end) }.into(), <start:@L> StartModule <body:Program> <end:@R> => ast::ModModule { body, type_ignores: thin_vec![], range: optional_range(start, end) }.into(),
<start:@L> StartInteractive <body:Program> <end:@R> => ast::ModInteractive { body, range: optional_range(start, end) }.into(), <start:@L> StartInteractive <body:Program> <end:@R> => ast::ModInteractive { body, range: optional_range(start, end) }.into(),
<start:@L> StartExpression <body:TestList> ("\n")* <end:@R> => ast::ModExpression { body: Box::new(body), range: optional_range(start, end) }.into() <start:@L> StartExpression <body:TestList> ("\n")* <end:@R> => ast::ModExpression { body: Box::new(body), range: optional_range(start, end) }.into()
}; };
Program: ast::Suite = { Program: ast::Suite = {
<lines:FileLine*> => { <lines:FileLine*> => {
lines.into_iter().flatten().collect() lines.into_iter().flatten().collect::<ThinVec<_>>()
}, },
}; };
// A file line either has a declaration, or an empty newline: // A file line either has a declaration, or an empty newline:
FileLine: ast::Suite = { FileLine: ast::Suite = {
Statement, Statement,
"\n" => vec![], "\n" => thin_vec![],
}; };
Suite: ast::Suite = { Suite: ast::Suite = {
SimpleStatement, SimpleStatement,
"\n" Indent <s:Statement+> Dedent => s.into_iter().flatten().collect(), "\n" Indent <s:OneOrMore<Statement>> Dedent => s.into_iter().flatten().collect(),
}; };
Statement: ast::Suite = { Statement: ast::Suite = {
SimpleStatement, SimpleStatement,
<s:CompoundStatement> => vec![s], <s:CompoundStatement> => thin_vec![s],
}; };
SimpleStatement: ast::Suite = { SimpleStatement: ast::Suite = {
<s1:SmallStatement> <s2:(";" SmallStatement)*> ";"? "\n" => { <s:OneOrMoreSeparated<SmallStatement, ";">> ";"? "\n" => {
let mut statements = vec![s1]; s
statements.extend(s2.into_iter().map(|e| e.1));
statements
} }
}; };
@ -88,7 +87,7 @@ ExpressionStatement: ast::Stmt = {
ast::StmtExpr { value: Box::new(expression), range: (location..end_location).into() } ast::StmtExpr { value: Box::new(expression), range: (location..end_location).into() }
) )
} else { } else {
let mut targets = vec![set_context(expression, ast::ExprContext::Store)]; let mut targets = thin_vec![set_context(expression, ast::ExprContext::Store)];
let mut values = suffix; let mut values = suffix;
while values.len() > 1 { while values.len() > 1 {
@ -207,7 +206,7 @@ RaiseStatement: ast::Stmt = {
}; };
ImportStatement: ast::Stmt = { ImportStatement: ast::Stmt = {
<location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => { <location:@L> "import" <names: OneOrMoreSeparated<ImportAsAlias<DottedName>, ",">> <end_location:@R> => {
ast::Stmt::Import( ast::Stmt::Import(
ast::StmtImport { names, range: (location..end_location).into() } ast::StmtImport { names, range: (location..end_location).into() }
) )
@ -239,12 +238,12 @@ ImportDots: ast::Int = {
"." => ast::Int::new(1), "." => ast::Int::new(1),
}; };
ImportAsNames: Vec<ast::Alias> = { ImportAsNames: ThinVec<ast::Alias> = {
<location:@L> <i:OneOrMore<ImportAsAlias<Identifier>>> <end_location:@R> => i, <location:@L> <i:OneOrMoreSeparated<ImportAsAlias<Identifier>, ",">> <end_location:@R> => i,
<location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" <end_location:@R> => i, <location:@L> "(" <i:OneOrMoreSeparated<ImportAsAlias<Identifier>, ",">> ","? ")" <end_location:@R> => i,
<location:@L> "*" <end_location:@R> => { <location:@L> "*" <end_location:@R> => {
// Star import all // Star import all
vec![ast::Alias { name: ast::Identifier::new("*"), asname: None, range: (location..end_location).into() }] thin_vec![ast::Alias { name: ast::Identifier::new("*"), asname: None, range: (location..end_location).into() }]
}, },
}; };
@ -268,7 +267,7 @@ DottedName: ast::Identifier = {
}; };
GlobalStatement: ast::Stmt = { GlobalStatement: ast::Stmt = {
<location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => { <location:@L> "global" <names:OneOrMoreSeparated<Identifier, ",">> <end_location:@R> => {
ast::Stmt::Global( ast::Stmt::Global(
ast::StmtGlobal { names, range: (location..end_location).into() } ast::StmtGlobal { names, range: (location..end_location).into() }
) )
@ -276,7 +275,7 @@ GlobalStatement: ast::Stmt = {
}; };
NonlocalStatement: ast::Stmt = { NonlocalStatement: ast::Stmt = {
<location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => { <location:@L> "nonlocal" <names:OneOrMoreSeparated<Identifier, ",">> <end_location:@R> => {
ast::Stmt::Nonlocal( ast::Stmt::Nonlocal(
ast::StmtNonlocal { names, range: (location..end_location).into() } ast::StmtNonlocal { names, range: (location..end_location).into() }
) )
@ -307,7 +306,7 @@ CompoundStatement: ast::Stmt = {
}; };
MatchStatement: ast::Stmt = { MatchStatement: ast::Stmt = {
<location:@L> "match" <subject:TestOrStarNamedExpr> ":" "\n" Indent <cases:MatchCase+> Dedent => { <location:@L> "match" <subject:TestOrStarNamedExpr> ":" "\n" Indent <cases:OneOrMore<MatchCase>> Dedent => {
let end_location = cases let end_location = cases
.last() .last()
.unwrap() .unwrap()
@ -323,7 +322,7 @@ MatchStatement: ast::Stmt = {
} }
) )
}, },
<location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:MatchCase+> Dedent => { <location:@L> "match" <subject:TestOrStarNamedExpr> "," ":" "\n" Indent <cases:OneOrMore<MatchCase>> Dedent => {
let end_location = cases let end_location = cases
.last() .last()
.unwrap() .unwrap()
@ -339,7 +338,7 @@ MatchStatement: ast::Stmt = {
} }
) )
}, },
<location:@L> "match" <subject:TestOrStarNamedExpr> "," <subjects:OneOrMore<TestOrStarNamedExpr>> ","? ":" "\n" Indent <cases:MatchCase+> Dedent => { <location:@L> "match" <subject:TestOrStarNamedExpr> "," <subjects:OneOrMoreSeparated<TestOrStarNamedExpr, ",">> ","? ":" "\n" Indent <cases:OneOrMore<MatchCase>> Dedent => {
let end_location = cases let end_location = cases
.last() .last()
.unwrap() .unwrap()
@ -385,11 +384,11 @@ Guard: ast::Expr = {
Patterns: ast::Pattern = { Patterns: ast::Pattern = {
<location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::MatchSequence( <location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::MatchSequence(
ast::PatternMatchSequence { ast::PatternMatchSequence {
patterns: vec![pattern], patterns: thin_vec![pattern],
range: (location..end_location).into() range: (location..end_location).into()
}, },
), ),
<location:@L> <pattern:Pattern> "," <patterns:OneOrMore<Pattern>> ","? <end_location:@R> => { <location:@L> <pattern:Pattern> "," <patterns:OneOrMoreSeparated<Pattern, ",">> ","? <end_location:@R> => {
let mut patterns = patterns; let mut patterns = patterns;
patterns.insert(0, pattern); patterns.insert(0, pattern);
ast::Pattern::MatchSequence( ast::Pattern::MatchSequence(
@ -428,7 +427,7 @@ AsPattern: ast::Pattern = {
OrPattern: ast::Pattern = { OrPattern: ast::Pattern = {
<pattern:ClosedPattern> => pattern, <pattern:ClosedPattern> => pattern,
<location:@L> <pattern:ClosedPattern> <patterns:("|" <ClosedPattern>)+> <end_location:@R> => { <location:@L> <pattern:ClosedPattern> <patterns:OneOrMore<("|" <ClosedPattern>)>> <end_location:@R> => {
let mut patterns = patterns; let mut patterns = patterns;
patterns.insert(0, pattern); patterns.insert(0, pattern);
ast::Pattern::MatchOr( ast::Pattern::MatchOr(
@ -451,7 +450,7 @@ SequencePattern: ast::Pattern = {
// A single-item tuple is a special case: it's a group pattern, _not_ a sequence pattern. // A single-item tuple is a special case: it's a group pattern, _not_ a sequence pattern.
<location:@L> "(" <pattern:Pattern> ")" <end_location:@R> => pattern, <location:@L> "(" <pattern:Pattern> ")" <end_location:@R> => pattern,
<location:@L> "(" ")" <end_location:@R> => ast::PatternMatchSequence { <location:@L> "(" ")" <end_location:@R> => ast::PatternMatchSequence {
patterns: vec![], patterns: thin_vec![],
range: (location..end_location).into() range: (location..end_location).into()
}.into(), }.into(),
<location:@L> "(" <pattern:Pattern> "," <patterns:Comma<Pattern>> ")" <end_location:@R> => { <location:@L> "(" <pattern:Pattern> "," <patterns:Comma<Pattern>> ")" <end_location:@R> => {
@ -605,13 +604,13 @@ MatchMappingEntry: (ast::Expr, ast::Pattern) = {
MappingPattern: ast::Pattern = { MappingPattern: ast::Pattern = {
<location:@L> "{" "}" <end_location:@R> => { <location:@L> "{" "}" <end_location:@R> => {
return ast::PatternMatchMapping { return ast::PatternMatchMapping {
keys: vec![], keys: thin_vec![],
patterns: vec![], patterns: thin_vec![],
rest: None, rest: None,
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into();
}, },
<location:@L> "{" <e:OneOrMore<MatchMappingEntry>> ","? "}" <end_location:@R> => { <location:@L> "{" <e:OneOrMoreSeparated<MatchMappingEntry, ",">> ","? "}" <end_location:@R> => {
let (keys, patterns) = e let (keys, patterns) = e
.into_iter() .into_iter()
.unzip(); .unzip();
@ -624,13 +623,13 @@ MappingPattern: ast::Pattern = {
}, },
<location:@L> "{" "**" <rest:Identifier> ","? "}" <end_location:@R> => { <location:@L> "{" "**" <rest:Identifier> ","? "}" <end_location:@R> => {
return ast::PatternMatchMapping { return ast::PatternMatchMapping {
keys: vec![], keys: thin_vec![],
patterns: vec![], patterns: thin_vec![],
rest: Some(rest), rest: Some(rest),
range: (location..end_location).into() range: (location..end_location).into()
}.into(); }.into();
}, },
<location:@L> "{" <e:OneOrMore<MatchMappingEntry>> "," "**" <rest:Identifier> ","? "}" <end_location:@R> => { <location:@L> "{" <e:OneOrMoreSeparated<MatchMappingEntry, ",">> "," "**" <rest:Identifier> ","? "}" <end_location:@R> => {
let (keys, patterns) = e let (keys, patterns) = e
.into_iter() .into_iter()
.unzip(); .unzip();
@ -648,7 +647,7 @@ MatchKeywordEntry: (ast::Identifier, ast::Pattern) = {
}; };
ClassPattern: ast::Pattern = { ClassPattern: ast::Pattern = {
<location:@L> <e:MatchName> "(" <patterns: OneOrMore<Pattern>> "," <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => { <location:@L> <e:MatchName> "(" <patterns: OneOrMoreSeparated<Pattern, ",">> "," <kwds:OneOrMoreSeparated<MatchKeywordEntry, ",">> ","? ")" <end_location:@R> => {
let (kwd_attrs, kwd_patterns) = kwds let (kwd_attrs, kwd_patterns) = kwds
.into_iter() .into_iter()
.unzip(); .unzip();
@ -660,22 +659,22 @@ ClassPattern: ast::Pattern = {
range: (location..end_location).into() range: (location..end_location).into()
}.into() }.into()
}, },
<location:@L> <e:MatchName> "(" <patterns: OneOrMore<Pattern>> ","? ")" <end_location:@R> => { <location:@L> <e:MatchName> "(" <patterns: OneOrMoreSeparated<Pattern, ",">> ","? ")" <end_location:@R> => {
ast::PatternMatchClass { ast::PatternMatchClass {
cls: Box::new(e), cls: Box::new(e),
patterns, patterns,
kwd_attrs: vec![], kwd_attrs: thin_vec![],
kwd_patterns: vec![], kwd_patterns: thin_vec![],
range: (location..end_location).into() range: (location..end_location).into()
}.into() }.into()
}, },
<location:@L> <e:MatchName> "(" <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => { <location:@L> <e:MatchName> "(" <kwds:OneOrMoreSeparated<MatchKeywordEntry, ",">> ","? ")" <end_location:@R> => {
let (kwd_attrs, kwd_patterns) = kwds let (kwd_attrs, kwd_patterns) = kwds
.into_iter() .into_iter()
.unzip(); .unzip();
ast::PatternMatchClass { ast::PatternMatchClass {
cls: Box::new(e), cls: Box::new(e),
patterns: vec![], patterns: thin_vec![],
kwd_attrs, kwd_attrs,
kwd_patterns, kwd_patterns,
range: (location..end_location).into() range: (location..end_location).into()
@ -684,13 +683,13 @@ ClassPattern: ast::Pattern = {
<location:@L> <e:MatchName> "(" ")" <end_location:@R> => { <location:@L> <e:MatchName> "(" ")" <end_location:@R> => {
ast::PatternMatchClass { ast::PatternMatchClass {
cls: Box::new(e), cls: Box::new(e),
patterns: vec![], patterns: thin_vec![],
kwd_attrs: vec![], kwd_attrs: thin_vec![],
kwd_patterns: vec![], kwd_patterns: thin_vec![],
range: (location..end_location).into() range: (location..end_location).into()
}.into() }.into()
}, },
<location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMore<Pattern>> "," <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => { <location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMoreSeparated<Pattern, ",">> "," <kwds:OneOrMoreSeparated<MatchKeywordEntry, ",">> ","? ")" <end_location:@R> => {
let (kwd_attrs, kwd_patterns) = kwds let (kwd_attrs, kwd_patterns) = kwds
.into_iter() .into_iter()
.unzip(); .unzip();
@ -702,22 +701,22 @@ ClassPattern: ast::Pattern = {
range: (location..end_location).into() range: (location..end_location).into()
}.into() }.into()
}, },
<location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMore<Pattern>> ","? ")" <end_location:@R> => { <location:@L> <e:MatchNameOrAttr> "(" <patterns: OneOrMoreSeparated<Pattern, ",">> ","? ")" <end_location:@R> => {
ast::PatternMatchClass { ast::PatternMatchClass {
cls: Box::new(e), cls: Box::new(e),
patterns, patterns,
kwd_attrs: vec![], kwd_attrs: thin_vec![],
kwd_patterns: vec![], kwd_patterns: thin_vec![],
range: (location..end_location).into() range: (location..end_location).into()
}.into() }.into()
}, },
<location:@L> <e:MatchNameOrAttr> "(" <kwds:OneOrMore<MatchKeywordEntry>> ","? ")" <end_location:@R> => { <location:@L> <e:MatchNameOrAttr> "(" <kwds:OneOrMoreSeparated<MatchKeywordEntry, ",">> ","? ")" <end_location:@R> => {
let (kwd_attrs, kwd_patterns) = kwds let (kwd_attrs, kwd_patterns) = kwds
.into_iter() .into_iter()
.unzip(); .unzip();
ast::PatternMatchClass { ast::PatternMatchClass {
cls: Box::new(e), cls: Box::new(e),
patterns: vec![], patterns: thin_vec![],
kwd_attrs, kwd_attrs,
kwd_patterns, kwd_patterns,
range: (location..end_location).into() range: (location..end_location).into()
@ -726,9 +725,9 @@ ClassPattern: ast::Pattern = {
<location:@L> <e:MatchNameOrAttr> "(" ")" <end_location:@R> => { <location:@L> <e:MatchNameOrAttr> "(" ")" <end_location:@R> => {
ast::PatternMatchClass { ast::PatternMatchClass {
cls: Box::new(e), cls: Box::new(e),
patterns: vec![], patterns: thin_vec![],
kwd_attrs: vec![], kwd_attrs: thin_vec![],
kwd_patterns: vec![], kwd_patterns: thin_vec![],
range: (location..end_location).into() range: (location..end_location).into()
}.into() }.into()
}, },
@ -749,7 +748,7 @@ IfStatement: ast::Stmt = {
let x = ast::Stmt::If( let x = ast::Stmt::If(
ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last, range: (i.0..end_location).into() } ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last, range: (i.0..end_location).into() }
); );
last = vec![x]; last = thin_vec![x];
} }
ast::Stmt::If( ast::Stmt::If(
@ -797,7 +796,7 @@ ForStatement: ast::Stmt = {
}; };
TryStatement: ast::Stmt = { TryStatement: ast::Stmt = {
<location:@L> "try" ":" <body:Suite> <handlers:ExceptClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => { <location:@L> "try" ":" <body:Suite> <handlers:OneOrMore<ExceptClause>> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
let orelse = else_suite.map(|s| s.2).unwrap_or_default(); let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default(); let finalbody = finally.map(|s| s.2).unwrap_or_default();
let end_location = finalbody let end_location = finalbody
@ -816,7 +815,7 @@ TryStatement: ast::Stmt = {
}, },
) )
}, },
<location:@L> "try" ":" <body:Suite> <handlers:ExceptStarClause+> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => { <location:@L> "try" ":" <body:Suite> <handlers:OneOrMore<ExceptStarClause>> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> <end_location:@R> => {
let orelse = else_suite.map(|s| s.2).unwrap_or_default(); let orelse = else_suite.map(|s| s.2).unwrap_or_default();
let finalbody = finally.map(|s| s.2).unwrap_or_default(); let finalbody = finally.map(|s| s.2).unwrap_or_default();
let end_location = finalbody let end_location = finalbody
@ -836,8 +835,8 @@ TryStatement: ast::Stmt = {
) )
}, },
<location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => { <location:@L> "try" ":" <body:Suite> <finally:("finally" ":" Suite)> => {
let handlers = vec![]; let handlers = thin_vec![];
let orelse = vec![]; let orelse = thin_vec![];
let finalbody = finally.2; let finalbody = finally.2;
let end_location = finalbody.last().unwrap().end(); let end_location = finalbody.last().unwrap().end();
ast::Stmt::Try( ast::Stmt::Try(
@ -915,20 +914,20 @@ WithStatement: ast::Stmt = {
}, },
}; };
WithItems: Vec<ast::Withitem> = { WithItems: ThinVec<ast::Withitem> = {
"(" <WithItemsNoAs> ","? ")", "(" <WithItemsNoAs> ","? ")",
"(" <left:(<WithItemsNoAs> ",")?> <mid:WithItem<"as">> <right:("," <WithItem<"all">>)*> ","? ")" => { "(" <left:(<WithItemsNoAs> ",")?> <mid:WithItem<"as">> <right:("," <WithItem<"all">>)*> ","? ")" => {
left.into_iter().flatten().chain([mid]).chain(right).collect() left.into_iter().flatten().chain([mid]).chain(right).collect()
}, },
<WithItem<"no-withitems">> => vec![<>], <WithItem<"no-withitems">> => thin_vec![<>],
<item:WithItem<"all">> <items:("," <WithItem<"all">>)+> => { <item:WithItem<"all">> <items:("," <WithItem<"all">>)+> => {
[item].into_iter().chain(items).collect() [item].into_iter().chain(items).collect()
} }
}; };
#[inline] #[inline]
WithItemsNoAs: Vec<ast::Withitem> = { WithItemsNoAs: ThinVec<ast::Withitem> = {
<location:@L> <all:OneOrMore<Test<"all">>> <end_location:@R> => { <location:@L> <all:OneOrMoreSeparated<Test<"all">, ",">> <end_location:@R> => {
all.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None, range: optional_range(location, end_location) }).collect() all.into_iter().map(|context_expr| ast::Withitem { context_expr, optional_vars: None, range: optional_range(location, end_location) }).collect()
}, },
} }
@ -942,7 +941,7 @@ WithItem<Goal>: ast::Withitem = {
}; };
FuncDef: ast::Stmt = { FuncDef: ast::Stmt = {
<decorator_list:Decorator*> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test<"all">)?> ":" <body:Suite> => { <decorator_list:ZeroOrMore<Decorator>> <location:@L> <is_async:"async"?> "def" <name:Identifier> <args:Parameters> <r:("->" Test<"all">)?> ":" <body:Suite> => {
let args = Box::new(args); let args = Box::new(args);
let returns = r.map(|x| Box::new(x.1)); let returns = r.map(|x| Box::new(x.1));
let end_location = body.last().unwrap().end(); let end_location = body.last().unwrap().end();
@ -959,13 +958,13 @@ Parameters: ast::Arguments = {
<location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter>)?> ")" <end_location:@R> =>? { <location:@L> "(" <a: (ParameterList<TypedParameter, StarTypedParameter>)?> ")" <end_location:@R> =>? {
let args = validate_arguments( let args = validate_arguments(
a.unwrap_or_else(|| ast::Arguments { a.unwrap_or_else(|| ast::Arguments {
posonlyargs: vec![], posonlyargs: thin_vec![],
args: vec![], args: thin_vec![],
vararg: None, vararg: None,
kwonlyargs: vec![], kwonlyargs: thin_vec![],
kw_defaults: vec![], kw_defaults: thin_vec![],
kwarg: None, kwarg: None,
defaults: vec![], defaults: thin_vec![],
range: optional_range(location, end_location) range: optional_range(location, end_location)
}) })
)?; )?;
@ -981,7 +980,7 @@ ParameterList<ArgType, StarArgType>: ast::Arguments = {
let (posonlyargs, args, defaults) = parse_params(param1)?; let (posonlyargs, args, defaults) = parse_params(param1)?;
// Now gather rest of parameters: // Now gather rest of parameters:
let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1); let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, thin_vec![], thin_vec![], None), |x| x.1);
Ok(ast::Arguments { Ok(ast::Arguments {
posonlyargs, posonlyargs,
@ -999,8 +998,8 @@ ParameterList<ArgType, StarArgType>: ast::Arguments = {
// Now gather rest of parameters: // Now gather rest of parameters:
let vararg = None; let vararg = None;
let kwonlyargs = vec![]; let kwonlyargs = thin_vec![];
let kw_defaults = vec![]; let kw_defaults = thin_vec![];
let kwarg = kw.1; let kwarg = kw.1;
Ok(ast::Arguments { Ok(ast::Arguments {
@ -1017,25 +1016,25 @@ ParameterList<ArgType, StarArgType>: ast::Arguments = {
<location:@L> <params:ParameterListStarArgs<ArgType, StarArgType>> ","? <end_location:@R> => { <location:@L> <params:ParameterListStarArgs<ArgType, StarArgType>> ","? <end_location:@R> => {
let (vararg, kwonlyargs, kw_defaults, kwarg) = params; let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
ast::Arguments { ast::Arguments {
posonlyargs: vec![], posonlyargs: thin_vec![],
args: vec![], args: thin_vec![],
kwonlyargs, kwonlyargs,
vararg, vararg,
kwarg, kwarg,
defaults: vec![], defaults: thin_vec![],
kw_defaults, kw_defaults,
range: optional_range(location, end_location) range: optional_range(location, end_location)
} }
}, },
<location:@L> <kwarg:KwargParameter<ArgType>> ","? <end_location:@R> => { <location:@L> <kwarg:KwargParameter<ArgType>> ","? <end_location:@R> => {
ast::Arguments { ast::Arguments {
posonlyargs: vec![], posonlyargs: thin_vec![],
args: vec![], args: thin_vec![],
kwonlyargs: vec![], kwonlyargs: thin_vec![],
vararg: None, vararg: None,
kwarg, kwarg,
defaults: vec![], defaults: thin_vec![],
kw_defaults: vec![], kw_defaults: thin_vec![],
range: optional_range(location, end_location) range: optional_range(location, end_location)
} }
}, },
@ -1043,11 +1042,11 @@ ParameterList<ArgType, StarArgType>: ast::Arguments = {
// Use inline here to make sure the "," is not creating an ambiguity. // Use inline here to make sure the "," is not creating an ambiguity.
#[inline] #[inline]
ParameterDefs<ArgType>: (Vec<(ast::Arg, Option<ast::Expr>)>, Vec<(ast::Arg, Option<ast::Expr>)>) = { ParameterDefs<ArgType>: (ThinVec<(ast::Arg, Option<ast::Expr>)>, ThinVec<(ast::Arg, Option<ast::Expr>)>) = {
<args:OneOrMore<ParameterDef<ArgType>>> => { <args:OneOrMoreSeparated<ParameterDef<ArgType>, ",">> => {
(vec![], args) (thin_vec![], args)
}, },
<pos_args:OneOrMore<ParameterDef<ArgType>>> "," "/" <args:("," ParameterDef<ArgType>)*> => { <pos_args:OneOrMoreSeparated<ParameterDef<ArgType>, ",">> "," "/" <args:("," ParameterDef<ArgType>)*> => {
(pos_args, args.into_iter().map(|e| e.1).collect()) (pos_args, args.into_iter().map(|e| e.1).collect())
}, },
}; };
@ -1078,12 +1077,12 @@ StarTypedParameter: ast::Arg = {
// Use inline here to make sure the "," is not creating an ambiguity. // Use inline here to make sure the "," is not creating an ambiguity.
// TODO: figure out another grammar that makes this inline no longer required. // TODO: figure out another grammar that makes this inline no longer required.
#[inline] #[inline]
ParameterListStarArgs<ArgType, StarArgType>: (Option<Box<ast::Arg>>, Vec<ast::Arg>, Vec<ast::Expr>, Option<Box<ast::Arg>>) = { ParameterListStarArgs<ArgType, StarArgType>: (Option<Box<ast::Arg>>, ThinVec<ast::Arg>, ThinVec<ast::Expr>, Option<Box<ast::Arg>>) = {
<location:@L> "*" <va:StarArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> =>? { <location:@L> "*" <va:StarArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> =>? {
// Extract keyword arguments: // Extract keyword arguments:
let mut kwonlyargs = Vec::new(); let mut kwonlyargs = Vec::new();
let mut kw_defaults = Vec::new(); let mut kw_defaults = ThinVec::new();
let mut kwargs = Vec::new(); let mut kwargs = ThinVec::new();
for (name, value) in kw.into_iter().map(|x| x.1) { for (name, value) in kw.into_iter().map(|x| x.1) {
if let Some(value) = value { if let Some(value) = value {
kwonlyargs.push(name); kwonlyargs.push(name);
@ -1115,10 +1114,10 @@ KwargParameter<ArgType>: Option<Box<ast::Arg>> = {
}; };
ClassDef: ast::Stmt = { ClassDef: ast::Stmt = {
<decorator_list:Decorator*> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => { <decorator_list:ZeroOrMore<Decorator>> <location:@L> "class" <name:Identifier> <a:("(" ArgumentList ")")?> ":" <body:Suite> => {
let (bases, keywords) = match a { let (bases, keywords) = match a {
Some((_, arg, _)) => (arg.args, arg.keywords), Some((_, arg, _)) => (arg.args, arg.keywords),
None => (vec![], vec![]), None => (thin_vec![], thin_vec![]),
}; };
let end_location = body.last().unwrap().end(); let end_location = body.last().unwrap().end();
ast::Stmt::ClassDef( ast::Stmt::ClassDef(
@ -1187,13 +1186,13 @@ LambdaDef: ast::Expr = {
let p = validate_arguments( let p = validate_arguments(
p.unwrap_or_else(|| { p.unwrap_or_else(|| {
ast::Arguments { ast::Arguments {
posonlyargs: vec![], posonlyargs: thin_vec![],
args: vec![], args: thin_vec![],
vararg: None, vararg: None,
kwonlyargs: vec![], kwonlyargs: thin_vec![],
kw_defaults: vec![], kw_defaults: thin_vec![],
kwarg: None, kwarg: None,
defaults: vec![], defaults: thin_vec![],
range: optional_range(location, end_location) range: optional_range(location, end_location)
} }
} }
@ -1211,7 +1210,7 @@ LambdaDef: ast::Expr = {
OrTest<Goal>: ast::Expr = { OrTest<Goal>: ast::Expr = {
<location:@L> <e1:AndTest<"all">> <e2:("or" AndTest<"all">)+> <end_location:@R> => { <location:@L> <e1:AndTest<"all">> <e2:("or" AndTest<"all">)+> <end_location:@R> => {
let mut values = vec![e1]; let mut values = thin_vec![e1];
values.extend(e2.into_iter().map(|e| e.1)); values.extend(e2.into_iter().map(|e| e.1));
ast::Expr::BoolOp( ast::Expr::BoolOp(
ast::ExprBoolOp { op: ast::Boolop::Or, values, range: (location..end_location).into() } ast::ExprBoolOp { op: ast::Boolop::Or, values, range: (location..end_location).into() }
@ -1222,7 +1221,7 @@ OrTest<Goal>: ast::Expr = {
AndTest<Goal>: ast::Expr = { AndTest<Goal>: ast::Expr = {
<location:@L> <e1:NotTest<"all">> <e2:("and" NotTest<"all">)+> <end_location:@R> => { <location:@L> <e1:NotTest<"all">> <e2:("and" NotTest<"all">)+> <end_location:@R> => {
let mut values = vec![e1]; let mut values = thin_vec![e1];
values.extend(e2.into_iter().map(|e| e.1)); values.extend(e2.into_iter().map(|e| e.1));
ast::Expr::BoolOp( ast::Expr::BoolOp(
ast::ExprBoolOp { op: ast::Boolop::And, values, range: (location..end_location).into() } ast::ExprBoolOp { op: ast::Boolop::And, values, range: (location..end_location).into() }
@ -1370,7 +1369,7 @@ SubscriptList: ast::Expr = {
if s2.is_empty() && trailing_comma.is_none() { if s2.is_empty() && trailing_comma.is_none() {
s1 s1
} else { } else {
let mut dims = vec![s1]; let mut dims = thin_vec![s1];
for x in s2 { for x in s2 {
dims.push(x.1) dims.push(x.1)
} }
@ -1417,7 +1416,7 @@ Atom<Goal>: ast::Expr = {
ast::ExprListComp { elt: Box::new(elt), generators, range: (location..end_location).into() } ast::ExprListComp { elt: Box::new(elt), generators, range: (location..end_location).into() }
) )
}, },
<location:@L> "(" <elts:OneOrMore<Test<"all">>> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => { <location:@L> "(" <elts:OneOrMoreSeparated<Test<"all">, ",">> <trailing_comma:","?> ")" <end_location:@R> if Goal != "no-withitems" => {
if elts.len() == 1 && trailing_comma.is_none() { if elts.len() == 1 && trailing_comma.is_none() {
elts.into_iter().next().unwrap() elts.into_iter().next().unwrap()
} else { } else {
@ -1426,7 +1425,7 @@ Atom<Goal>: ast::Expr = {
) )
} }
}, },
<location:@L> "(" <left:(<OneOrMore<Test<"all">>> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? { <location:@L> "(" <left:(<OneOrMoreSeparated<Test<"all">, ",">> ",")?> <mid:NamedOrStarExpr> <right:("," <TestOrStarNamedExpr>)*> <trailing_comma:","?> ")" <end_location:@R> =>? {
if left.is_none() && right.is_empty() && trailing_comma.is_none() { if left.is_none() && right.is_empty() && trailing_comma.is_none() {
if mid.is_starred_expr() { if mid.is_starred_expr() {
Err(LexicalError{ Err(LexicalError{
@ -1443,7 +1442,7 @@ Atom<Goal>: ast::Expr = {
} }
}, },
<location:@L> "(" ")" <end_location:@R> => ast::Expr::Tuple( <location:@L> "(" ")" <end_location:@R> => ast::Expr::Tuple(
ast::ExprTuple { elts: Vec::new(), ctx: ast::ExprContext::Load, range: (location..end_location).into() } ast::ExprTuple { elts: ThinVec::new(), ctx: ast::ExprContext::Load, range: (location..end_location).into() }
), ),
"(" <e:YieldExpr> ")" => e, "(" <e:YieldExpr> ")" => e,
<location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => { <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
@ -1491,12 +1490,12 @@ Atom<Goal>: ast::Expr = {
<location:@L> "..." <end_location:@R> => ast::Expr::Constant(ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None, range: (location..end_location).into() }), <location:@L> "..." <end_location:@R> => ast::Expr::Constant(ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None, range: (location..end_location).into() }),
}; };
ListLiteralValues: Vec<ast::Expr> = { ListLiteralValues: ThinVec<ast::Expr> = {
<e:OneOrMore<TestOrStarNamedExpr>> ","? => e, <e:OneOrMoreSeparated<TestOrStarNamedExpr, ",">> ","? => e,
}; };
DictLiteralValues: Vec<(Option<Box<ast::Expr>>, ast::Expr)> = { DictLiteralValues: ThinVec<(Option<Box<ast::Expr>>, ast::Expr)> = {
<elements:OneOrMore<DictElement>> ","? => elements, <elements:OneOrMoreSeparated<DictElement, ",">> ","? => elements,
}; };
DictEntry: (ast::Expr, ast::Expr) = { DictEntry: (ast::Expr, ast::Expr) = {
@ -1508,8 +1507,8 @@ DictElement: (Option<Box<ast::Expr>>, ast::Expr) = {
"**" <e:Expression<"all">> => (None, e), "**" <e:Expression<"all">> => (None, e),
}; };
SetLiteralValues: Vec<ast::Expr> = { SetLiteralValues: ThinVec<ast::Expr> = {
<e1:OneOrMore<TestOrStarNamedExpr>> ","? => e1 <e1:OneOrMoreSeparated<TestOrStarNamedExpr, ",">> ","? => e1
}; };
ExpressionOrStarExpression = { ExpressionOrStarExpression = {
@ -1521,8 +1520,8 @@ ExpressionList: ast::Expr = {
GenericList<ExpressionOrStarExpression> GenericList<ExpressionOrStarExpression>
}; };
ExpressionList2: Vec<ast::Expr> = { ExpressionList2: ThinVec<ast::Expr> = {
<elements:OneOrMore<ExpressionOrStarExpression>> ","? => elements, <elements:OneOrMoreSeparated<ExpressionOrStarExpression, ",">> ","? => elements,
}; };
// A test list is one of: // A test list is one of:
@ -1535,7 +1534,7 @@ TestList: ast::Expr = {
}; };
GenericList<Element>: ast::Expr = { GenericList<Element>: ast::Expr = {
<location:@L> <elts:OneOrMore<Element>> <trailing_comma:","?> <end_location:@R> => { <location:@L> <elts:OneOrMoreSeparated<Element, ",">> <trailing_comma:","?> <end_location:@R> => {
if elts.len() == 1 && trailing_comma.is_none() { if elts.len() == 1 && trailing_comma.is_none() {
elts.into_iter().next().unwrap() elts.into_iter().next().unwrap()
} else { } else {
@ -1554,10 +1553,10 @@ StarExpr: ast::Expr = {
}; };
// Comprehensions: // Comprehensions:
CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c; CompFor: ThinVec<ast::Comprehension> = <c:OneOrMore<SingleForComprehension>> => c;
SingleForComprehension: ast::Comprehension = { SingleForComprehension: ast::Comprehension = {
<location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest<"all">> <ifs:ComprehensionIf*> <end_location:@R> => { <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest<"all">> <ifs:ZeroOrMore<ComprehensionIf>> <end_location:@R> => {
let is_async = is_async.is_some(); let is_async = is_async.is_some();
ast::Comprehension { ast::Comprehension {
target: set_context(target, ast::ExprContext::Store), target: set_context(target, ast::ExprContext::Store),
@ -1604,22 +1603,36 @@ FunctionArgument: (Option<(TextSize, TextSize, Option<ast::Identifier>)>, ast::E
}; };
#[inline] #[inline]
Comma<T>: Vec<T> = { Comma<T>: ThinVec<T> = {
<items: (<T> ",")*> <last: T?> => { ","? => ThinVec::new(),
let mut items = items; <e:T> => thin_vec![e],
items.extend(last); <mut v: OneOrMoreSeparated<T, ",">> "," <e:T?> => {
items v.extend(e.into_iter());
v
}
};
OneOrMoreSeparated<T, S>: ThinVec<T> = {
<e:T> => thin_vec![e],
<mut v: OneOrMoreSeparated<T, S>> S <e:T> => {
v.push(e);
v
} }
}; };
#[inline] #[inline]
OneOrMore<T>: Vec<T> = { ZeroOrMore<T>: ThinVec<T> = {
<i1: T> <i2:("," T)*> => { => ThinVec::new(),
let mut items = vec![i1]; <v:OneOrMore<T>> => v
items.extend(i2.into_iter().map(|e| e.1)); }
items
OneOrMore<T>: ThinVec<T> = {
<e:T> => thin_vec![e],
<mut v:OneOrMore<T>> <e:T> => {
v.push(e);
v
} }
}; }
Constant: ast::Constant = { Constant: ast::Constant = {
<value:int> => ast::Constant::Int(value), <value:int> => ast::Constant::Int(value),

80506
parser/src/python.rs generated
View file

File diff suppressed because it is too large Load diff

11
parser/src/string.rs
View file

@ -15,6 +15,7 @@ use rustpython_parser_core::{
text_size::{TextLen, TextSize}, text_size::{TextLen, TextSize},
ConversionFlag, ConversionFlag,
}; };
use thin_vec::{thin_vec, ThinVec};
// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798 // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
const MAX_UNICODE_NAME: usize = 88; const MAX_UNICODE_NAME: usize = 88;
@ -398,8 +399,8 @@ impl<'a> StringParser<'a> {
Err(FStringError::new(UnclosedLbrace, self.get_pos()).into()) Err(FStringError::new(UnclosedLbrace, self.get_pos()).into())
} }
fn parse_spec(&mut self, nested: u8) -> Result<Vec<Expr>, LexicalError> { fn parse_spec(&mut self, nested: u8) -> Result<ThinVec<Expr>, LexicalError> {
let mut spec_constructor = Vec::new(); let mut spec_constructor = ThinVec::new();
let mut constant_piece = String::new(); let mut constant_piece = String::new();
while let Some(&next) = self.peek() { while let Some(&next) = self.peek() {
match next { match next {
@ -670,10 +671,10 @@ pub(crate) fn parse_strings(
} }
// De-duplicate adjacent constants. // De-duplicate adjacent constants.
let mut deduped: Vec<Expr> = vec![]; let mut deduped: ThinVec<Expr> = thin_vec![];
let mut current: Vec<String> = vec![]; let mut current: ThinVec<String> = thin_vec![];
let take_current = |current: &mut Vec<String>| -> Expr { let take_current = |current: &mut ThinVec<String>| -> Expr {
Expr::Constant(ast::ExprConstant { Expr::Constant(ast::ExprConstant {
value: Constant::Str(current.drain(..).join("")), value: Constant::Str(current.drain(..).join("")),
kind: initial_kind.clone(), kind: initial_kind.clone(),