Give identifier and int ast types

ast/asdl_rs.py

@@ -17,9 +17,9 @@ TABSIZE = 4
 AUTO_GEN_MESSAGE = "// File automatically generated by {}.\n\n"
 
 builtin_type_mapping = {
-    "identifier": "Ident",
+    "identifier": "Identifier",
     "string": "String",
-    "int": "u32",
+    "int": "Int",
     "constant": "Constant",
 }
 assert builtin_type_mapping.keys() == asdl.builtin_types

ast/src/attributed.rs

@@ -1,76 +0,0 @@
-use rustpython_parser_core::{
-    source_code::{SourceLocation, SourceRange},
-    text_size::{TextRange, TextSize},
-};
-
-#[derive(Clone, Debug, PartialEq)]
-pub struct Attributed<T, U = ()> {
-    pub range: TextRange,
-    pub custom: U,
-    pub node: T,
-}
-
-impl<T, U> Attributed<T, U> {
-    /// Returns the node
-    #[inline]
-    pub fn node(&self) -> &T {
-        &self.node
-    }
-
-    /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
-    #[inline]
-    pub const fn range(&self) -> TextRange {
-        self.range
-    }
-
-    /// Returns the absolute start position of the node from the beginning of the document.
-    #[inline]
-    pub const fn start(&self) -> TextSize {
-        self.range.start()
-    }
-
-    /// Returns the absolute position at which the node ends in the source document.
-    #[inline]
-    pub const fn end(&self) -> TextSize {
-        self.range.end()
-    }
-}
-
-impl<T> Attributed<T, ()> {
-    /// Creates a new node that spans the position specified by `range`.
-    pub fn new(range: impl Into<TextRange>, node: T) -> Self {
-        Self {
-            range: range.into(),
-            custom: (),
-            node,
-        }
-    }
-
-    /// Consumes self and returns the node.
-    #[inline]
-    pub fn into_node(self) -> T {
-        self.node
-    }
-}
-
-impl<T> Attributed<T, SourceRange> {
-    /// Returns the absolute start position of the node from the beginning of the document.
-    #[inline]
-    pub const fn location(&self) -> SourceLocation {
-        self.custom.start
-    }
-
-    /// Returns the absolute position at which the node ends in the source document.
-    #[inline]
-    pub const fn end_location(&self) -> Option<SourceLocation> {
-        self.custom.end
-    }
-}
-
-impl<T, U> std::ops::Deref for Attributed<T, U> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        &self.node
-    }
-}

ast/src/builtin.rs

@@ -0,0 +1,186 @@
+//! `builtin_types` in asdl.py and Attributed
+
+use num_bigint::BigInt;
+use rustpython_parser_core::text_size::{TextRange, TextSize};
+
+pub type String = std::string::String;
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct Identifier(String);
+
+impl Identifier {
+    #[inline]
+    pub fn new(s: impl Into<String>) -> Self {
+        Self(s.into())
+    }
+    #[inline]
+    pub fn as_str(&self) -> &str {
+        self.0.as_str()
+    }
+}
+
+impl std::string::ToString for Identifier {
+    #[inline]
+    fn to_string(&self) -> String {
+        self.0.clone()
+    }
+}
+
+impl From<Identifier> for String {
+    #[inline]
+    fn from(id: Identifier) -> String {
+        id.0
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Int(u32);
+
+impl Int {
+    pub fn new(i: u32) -> Self {
+        Self(i)
+    }
+    pub fn new_bool(i: bool) -> Self {
+        Self(i as u32)
+    }
+    pub fn to_u32(&self) -> u32 {
+        self.0
+    }
+    pub fn to_bool(&self) -> bool {
+        self.0 > 0
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum Constant {
+    None,
+    Bool(bool),
+    Str(String),
+    Bytes(Vec<u8>),
+    Int(BigInt),
+    Tuple(Vec<Constant>),
+    Float(f64),
+    Complex { real: f64, imag: f64 },
+    Ellipsis,
+}
+
+impl From<String> for Constant {
+    fn from(s: String) -> Constant {
+        Self::Str(s)
+    }
+}
+impl From<Vec<u8>> for Constant {
+    fn from(b: Vec<u8>) -> Constant {
+        Self::Bytes(b)
+    }
+}
+impl From<bool> for Constant {
+    fn from(b: bool) -> Constant {
+        Self::Bool(b)
+    }
+}
+impl From<BigInt> for Constant {
+    fn from(i: BigInt) -> Constant {
+        Self::Int(i)
+    }
+}
+
+#[cfg(feature = "rustpython-literal")]
+impl std::fmt::Display for Constant {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Constant::None => f.pad("None"),
+            Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
+            Constant::Str(s) => rustpython_literal::escape::UnicodeEscape::new_repr(s.as_str())
+                .str_repr()
+                .write(f),
+            Constant::Bytes(b) => {
+                let escape = rustpython_literal::escape::AsciiEscape::new_repr(b);
+                let repr = escape.bytes_repr().to_string().unwrap();
+                f.pad(&repr)
+            }
+            Constant::Int(i) => i.fmt(f),
+            Constant::Tuple(tup) => {
+                if let [elt] = &**tup {
+                    write!(f, "({elt},)")
+                } else {
+                    f.write_str("(")?;
+                    for (i, elt) in tup.iter().enumerate() {
+                        if i != 0 {
+                            f.write_str(", ")?;
+                        }
+                        elt.fmt(f)?;
+                    }
+                    f.write_str(")")
+                }
+            }
+            Constant::Float(fp) => f.pad(&rustpython_literal::float::to_string(*fp)),
+            Constant::Complex { real, imag } => {
+                if *real == 0.0 {
+                    write!(f, "{imag}j")
+                } else {
+                    write!(f, "({real}{imag:+}j)")
+                }
+            }
+            Constant::Ellipsis => f.pad("..."),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Attributed<T, U = ()> {
+    pub range: TextRange,
+    pub custom: U,
+    pub node: T,
+}
+
+impl<T, U> Attributed<T, U> {
+    /// Returns the node
+    #[inline]
+    pub fn node(&self) -> &T {
+        &self.node
+    }
+
+    /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
+    #[inline]
+    pub const fn range(&self) -> TextRange {
+        self.range
+    }
+
+    /// Returns the absolute start position of the node from the beginning of the document.
+    #[inline]
+    pub const fn start(&self) -> TextSize {
+        self.range.start()
+    }
+
+    /// Returns the absolute position at which the node ends in the source document.
+    #[inline]
+    pub const fn end(&self) -> TextSize {
+        self.range.end()
+    }
+}
+
+impl<T> Attributed<T, ()> {
+    /// Creates a new node that spans the position specified by `range`.
+    pub fn new(range: impl Into<TextRange>, node: T) -> Self {
+        Self {
+            range: range.into(),
+            custom: (),
+            node,
+        }
+    }
+
+    /// Consumes self and returns the node.
+    #[inline]
+    pub fn into_node(self) -> T {
+        self.node
+    }
+}
+
+impl<T, U> std::ops::Deref for Attributed<T, U> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.node
+    }
+}

ast/src/fold_helpers.rs

@@ -1,4 +1,4 @@
-use crate::{constant, fold::Fold};
+use crate::{builtin, fold::Fold};
 
 pub trait Foldable<T, U> {
     type Mapped;
@@ -62,4 +62,10 @@ macro_rules! simple_fold {
     };
 }
 
-simple_fold!(u32, String, bool, constant::Constant);
+simple_fold!(
+    builtin::Int,
+    builtin::String,
+    builtin::Identifier,
+    bool,
+    builtin::Constant
+);

ast/src/gen/generic.rs

@@ -56,7 +56,7 @@ pub enum Mod<U = ()> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtFunctionDef<U = ()> {
-    pub name: Ident,
+    pub name: Identifier,
     pub args: Box<Arguments<U>>,
     pub body: Vec<Stmt<U>>,
     pub decorator_list: Vec<Expr<U>>,
@@ -72,7 +72,7 @@ impl<U> From<StmtFunctionDef<U>> for StmtKind<U> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtAsyncFunctionDef<U = ()> {
-    pub name: Ident,
+    pub name: Identifier,
     pub args: Box<Arguments<U>>,
     pub body: Vec<Stmt<U>>,
     pub decorator_list: Vec<Expr<U>>,
@@ -88,7 +88,7 @@ impl<U> From<StmtAsyncFunctionDef<U>> for StmtKind<U> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtClassDef<U = ()> {
-    pub name: Ident,
+    pub name: Identifier,
     pub bases: Vec<Expr<U>>,
     pub keywords: Vec<Keyword<U>>,
     pub body: Vec<Stmt<U>>,
@@ -154,7 +154,7 @@ pub struct StmtAnnAssign<U = ()> {
     pub target: Box<Expr<U>>,
     pub annotation: Box<Expr<U>>,
     pub value: Option<Box<Expr<U>>>,
-    pub simple: u32,
+    pub simple: Int,
 }
 
 impl<U> From<StmtAnnAssign<U>> for StmtKind<U> {
@@ -322,9 +322,9 @@ impl<U> From<StmtImport<U>> for StmtKind<U> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtImportFrom<U = ()> {
-    pub module: Option<Ident>,
+    pub module: Option<Identifier>,
     pub names: Vec<Alias<U>>,
-    pub level: Option<u32>,
+    pub level: Option<Int>,
 }
 
 impl<U> From<StmtImportFrom<U>> for StmtKind<U> {
@@ -335,7 +335,7 @@ impl<U> From<StmtImportFrom<U>> for StmtKind<U> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtGlobal {
-    pub names: Vec<Ident>,
+    pub names: Vec<Identifier>,
 }
 
 impl From<StmtGlobal> for StmtKind {
@@ -346,7 +346,7 @@ impl From<StmtGlobal> for StmtKind {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct StmtNonlocal {
-    pub names: Vec<Ident>,
+    pub names: Vec<Identifier>,
 }
 
 impl From<StmtNonlocal> for StmtKind {
@@ -606,7 +606,7 @@ impl<U> From<ExprCall<U>> for ExprKind<U> {
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExprFormattedValue<U = ()> {
     pub value: Box<Expr<U>>,
-    pub conversion: u32,
+    pub conversion: Int,
     pub format_spec: Option<Box<Expr<U>>>,
 }
 
@@ -642,7 +642,7 @@ impl From<ExprConstant> for ExprKind {
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExprAttribute<U = ()> {
     pub value: Box<Expr<U>>,
-    pub attr: Ident,
+    pub attr: Identifier,
     pub ctx: ExprContext,
 }
 
@@ -679,7 +679,7 @@ impl<U> From<ExprStarred<U>> for ExprKind<U> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExprName {
-    pub id: Ident,
+    pub id: Identifier,
     pub ctx: ExprContext,
 }
 
@@ -815,13 +815,13 @@ pub struct Comprehension<U = ()> {
     pub target: Expr<U>,
     pub iter: Expr<U>,
     pub ifs: Vec<Expr<U>>,
-    pub is_async: u32,
+    pub is_async: Int,
 }
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExcepthandlerExceptHandler<U = ()> {
     pub type_: Option<Box<Expr<U>>>,
-    pub name: Option<Ident>,
+    pub name: Option<Identifier>,
     pub body: Vec<Stmt<U>>,
 }
 
@@ -850,7 +850,7 @@ pub struct Arguments<U = ()> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct ArgData<U = ()> {
-    pub arg: Ident,
+    pub arg: Identifier,
     pub annotation: Option<Box<Expr<U>>>,
     pub type_comment: Option<String>,
 }
@@ -858,15 +858,15 @@ pub type Arg<U = ()> = Attributed<ArgData<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct KeywordData<U = ()> {
-    pub arg: Option<Ident>,
+    pub arg: Option<Identifier>,
     pub value: Expr<U>,
 }
 pub type Keyword<U = ()> = Attributed<KeywordData<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct AliasData {
-    pub name: Ident,
-    pub asname: Option<Ident>,
+    pub name: Identifier,
+    pub asname: Option<Identifier>,
 }
 pub type Alias<U = ()> = Attributed<AliasData, U>;
 
@@ -920,7 +920,7 @@ impl<U> From<PatternMatchSequence<U>> for PatternKind<U> {
 pub struct PatternMatchMapping<U = ()> {
     pub keys: Vec<Expr<U>>,
     pub patterns: Vec<Pattern<U>>,
-    pub rest: Option<Ident>,
+    pub rest: Option<Identifier>,
 }
 
 impl<U> From<PatternMatchMapping<U>> for PatternKind<U> {
@@ -933,7 +933,7 @@ impl<U> From<PatternMatchMapping<U>> for PatternKind<U> {
 pub struct PatternMatchClass<U = ()> {
     pub cls: Box<Expr<U>>,
     pub patterns: Vec<Pattern<U>>,
-    pub kwd_attrs: Vec<Ident>,
+    pub kwd_attrs: Vec<Identifier>,
     pub kwd_patterns: Vec<Pattern<U>>,
 }
 
@@ -945,7 +945,7 @@ impl<U> From<PatternMatchClass<U>> for PatternKind<U> {
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct PatternMatchStar {
-    pub name: Option<Ident>,
+    pub name: Option<Identifier>,
 }
 
 impl From<PatternMatchStar> for PatternKind {
@@ -957,7 +957,7 @@ impl From<PatternMatchStar> for PatternKind {
 #[derive(Clone, Debug, PartialEq)]
 pub struct PatternMatchAs<U = ()> {
     pub pattern: Option<Box<Pattern<U>>>,
-    pub name: Option<Ident>,
+    pub name: Option<Identifier>,
 }
 
 impl<U> From<PatternMatchAs<U>> for PatternKind<U> {
@@ -992,7 +992,7 @@ pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct TypeIgnoreTypeIgnore {
-    pub lineno: u32,
+    pub lineno: Int,
     pub tag: String,
 }
 

ast/src/lib.rs

@@ -1,12 +1,9 @@
-mod attributed;
-mod constant;
+mod builtin;
 #[cfg(feature = "fold")]
 mod fold_helpers;
 mod generic {
     #![allow(clippy::derive_partial_eq_without_eq)]
-    pub use crate::{constant::*, Attributed};
-
-    type Ident = String;
+    pub use crate::builtin::*;
 
     include!("gen/generic.rs");
 }
@@ -16,8 +13,7 @@ mod source_locator;
 #[cfg(feature = "unparse")]
 mod unparse;
 
-pub use attributed::Attributed;
-pub use constant::Constant;
+pub use builtin::*;
 pub use generic::*;
 pub use rustpython_parser_core::{text_size, ConversionFlag};
 
@@ -44,3 +40,9 @@ pub mod located {
 pub use rustpython_parser_core::source_code;
 #[cfg(feature = "visitor")]
 pub use visitor::Visitor;
+
+#[cfg(feature = "constant-optimization")]
+mod optimizer;
+
+#[cfg(feature = "constant-optimization")]
+pub use optimizer::ConstantOptimizer;

ast/src/optimizer.rs

@@ -1,87 +1,9 @@
-use num_bigint::BigInt;
+use crate::builtin::Constant;
 
-#[derive(Clone, Debug, PartialEq)]
-pub enum Constant {
-    None,
-    Bool(bool),
-    Str(String),
-    Bytes(Vec<u8>),
-    Int(BigInt),
-    Tuple(Vec<Constant>),
-    Float(f64),
-    Complex { real: f64, imag: f64 },
-    Ellipsis,
-}
-
-impl From<String> for Constant {
-    fn from(s: String) -> Constant {
-        Self::Str(s)
-    }
-}
-impl From<Vec<u8>> for Constant {
-    fn from(b: Vec<u8>) -> Constant {
-        Self::Bytes(b)
-    }
-}
-impl From<bool> for Constant {
-    fn from(b: bool) -> Constant {
-        Self::Bool(b)
-    }
-}
-impl From<BigInt> for Constant {
-    fn from(i: BigInt) -> Constant {
-        Self::Int(i)
-    }
-}
-
-#[cfg(feature = "rustpython-literal")]
-impl std::fmt::Display for Constant {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Constant::None => f.pad("None"),
-            Constant::Bool(b) => f.pad(if *b { "True" } else { "False" }),
-            Constant::Str(s) => rustpython_literal::escape::UnicodeEscape::new_repr(s.as_str())
-                .str_repr()
-                .write(f),
-            Constant::Bytes(b) => {
-                let escape = rustpython_literal::escape::AsciiEscape::new_repr(b);
-                let repr = escape.bytes_repr().to_string().unwrap();
-                f.pad(&repr)
-            }
-            Constant::Int(i) => i.fmt(f),
-            Constant::Tuple(tup) => {
-                if let [elt] = &**tup {
-                    write!(f, "({elt},)")
-                } else {
-                    f.write_str("(")?;
-                    for (i, elt) in tup.iter().enumerate() {
-                        if i != 0 {
-                            f.write_str(", ")?;
-                        }
-                        elt.fmt(f)?;
-                    }
-                    f.write_str(")")
-                }
-            }
-            Constant::Float(fp) => f.pad(&rustpython_literal::float::to_string(*fp)),
-            Constant::Complex { real, imag } => {
-                if *real == 0.0 {
-                    write!(f, "{imag}j")
-                } else {
-                    write!(f, "({real}{imag:+}j)")
-                }
-            }
-            Constant::Ellipsis => f.pad("..."),
-        }
-    }
-}
-
-#[cfg(feature = "constant-optimization")]
 #[non_exhaustive]
 #[derive(Default)]
 pub struct ConstantOptimizer {}
 
-#[cfg(feature = "constant-optimization")]
 impl ConstantOptimizer {
     #[inline]
     pub fn new() -> Self {
@@ -135,7 +57,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
+    use num_bigint::BigInt;
     use rustpython_parser_core::text_size::TextRange;
 
     #[cfg(feature = "constant-optimization")]

ast/src/source_locator.rs

@@ -1,5 +1,5 @@
-use crate::attributed::Attributed;
-use rustpython_parser_core::source_code::{SourceLocator, SourceRange};
+use crate::builtin::Attributed;
+use rustpython_parser_core::source_code::{SourceLocation, SourceLocator, SourceRange};
 
 impl crate::fold::Fold<()> for SourceLocator<'_> {
     type TargetU = SourceRange;
@@ -23,3 +23,17 @@ impl crate::fold::Fold<()> for SourceLocator<'_> {
         })
     }
 }
+
+impl<T> Attributed<T, SourceRange> {
+    /// Returns the absolute start position of the node from the beginning of the document.
+    #[inline]
+    pub const fn location(&self) -> SourceLocation {
+        self.custom.start
+    }
+
+    /// Returns the absolute position at which the node ends in the source document.
+    #[inline]
+    pub const fn end_location(&self) -> Option<SourceLocation> {
+        self.custom.end
+    }
+}

ast/src/unparse.rs

@@ -1,5 +1,7 @@
 use crate::ConversionFlag;
-use crate::{Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, Expr, ExprKind, Operator};
+use crate::{
+    Arg, Arguments, Boolop, Cmpop, Comprehension, Constant, Expr, ExprKind, Identifier, Operator,
+};
 use std::fmt;
 
 mod precedence {
@@ -32,6 +34,9 @@ impl<'a> Unparser<'a> {
     fn p(&mut self, s: &str) -> fmt::Result {
         self.f.write_str(s)
     }
+    fn p_id(&mut self, s: &Identifier) -> fmt::Result {
+        self.f.write_str(s.as_str())
+    }
     fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result {
         if cond {
             self.f.write_str(s)?;
@@ -270,7 +275,7 @@ impl<'a> Unparser<'a> {
                     for kw in keywords {
                         self.p_delim(&mut first, ", ")?;
                         if let Some(arg) = &kw.node.arg {
-                            self.p(arg)?;
+                            self.p_id(arg)?;
                             self.p("=")?;
                         } else {
                             self.p("**")?;
@@ -284,7 +289,7 @@ impl<'a> Unparser<'a> {
                 value,
                 conversion,
                 format_spec,
-            }) => self.unparse_formatted(value, *conversion, format_spec.as_deref())?,
+            }) => self.unparse_formatted(value, conversion.to_u32(), format_spec.as_deref())?,
             ExprKind::JoinedStr(crate::ExprJoinedStr { values }) => {
                 self.unparse_joined_str(values, false)?
             }
@@ -316,7 +321,7 @@ impl<'a> Unparser<'a> {
                     "."
                 };
                 self.p(period)?;
-                self.p(attr)?;
+                self.p_id(attr)?;
             }
             ExprKind::Subscript(crate::ExprSubscript { value, slice, .. }) => {
                 self.unparse_expr(value, precedence::ATOM)?;
@@ -337,7 +342,7 @@ impl<'a> Unparser<'a> {
                 self.p("*")?;
                 self.unparse_expr(value, precedence::EXPR)?;
             }
-            ExprKind::Name(crate::ExprName { id, .. }) => self.p(id)?,
+            ExprKind::Name(crate::ExprName { id, .. }) => self.p_id(id)?,
             ExprKind::List(crate::ExprList { elts, .. }) => {
                 self.p("[")?;
                 let mut first = true;
@@ -415,7 +420,7 @@ impl<'a> Unparser<'a> {
         Ok(())
     }
     fn unparse_arg<U>(&mut self, arg: &Arg<U>) -> fmt::Result {
-        self.p(&arg.node.arg)?;
+        self.p_id(&arg.node.arg)?;
         if let Some(ann) = &arg.node.annotation {
             write!(self, ": {}", **ann)?;
         }
@@ -424,7 +429,7 @@ impl<'a> Unparser<'a> {
 
     fn unparse_comp<U>(&mut self, generators: &[Comprehension<U>]) -> fmt::Result {
         for comp in generators {
-            self.p(if comp.is_async > 0 {
+            self.p(if comp.is_async.to_bool() {
                 " async for "
             } else {
                 " for "
@@ -497,7 +502,7 @@ impl<'a> Unparser<'a> {
                 value,
                 conversion,
                 format_spec,
-            }) => self.unparse_formatted(value, *conversion, format_spec.as_deref()),
+            }) => self.unparse_formatted(value, conversion.to_u32(), format_spec.as_deref()),
             _ => unreachable!(),
         }
     }